diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index b51913a817917537f50e79a573d291267509850f..32105bcc164f7ae4f8a94c3b788db12ba6b22797 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -37,5 +37,7 @@ properties: - const: apple,s5l8960x-cluster-cpufreq - items: - - const: apple,t6020-cluster-cpufreq + - enum: + - apple,t6020-cluster-cpufreq + - apple,t8132-cluster-cpufreq - const: apple,t8112-cluster-cpufreq diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index 0e3780ad9dd79abbdb75d4c5a06300953ab61cc5..db5b2ab5deef37e2b34266cea86b74fd0870b509 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -24,4 +24,5 @@ properties: - apple,t6000-sio - apple,t8103-sio + - apple,t8132-sio - const: apple,sio diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index e179199dbd3b54a2752c95e0cdd8c6ac1c9efbcb..865edc6c49217f99ac7a0d1a014f06099adbb8e1 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -33,4 +33,5 @@ properties: - apple,t6020-dart - apple,t8122-dart + - apple,t8132-dart - const: apple,t8110-dart diff --git a/Documentation/devicetree/bindings/iommu/apple,sart.yaml b/Documentation/devicetree/bindings/iommu/apple,sart.yaml index 8fd7775704ab7d6e16ca4b4534fc798e13f6fdb9..a095e8d3a811ccb8a9de05553078a2895d768d2f 100644 --- a/Documentation/devicetree/bindings/iommu/apple,sart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,sart.yaml @@ -35,4 +35,5 @@ properties: - apple,t8112-sart - apple,t8122-sart + - apple,t8132-sart - const: apple,t6000-sart - enum: diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 946c909c6922757acb6731ef56c3348e9b15799f..8fda3068a8096bed09f691d8eeedc1b5a704fda0 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -32,4 +32,5 @@ properties: - apple,t8112-asc-mailbox - apple,t8122-asc-mailbox + - apple,t8132-asc-mailbox - apple,t6000-asc-mailbox - apple,t6020-asc-mailbox diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index d5d30c05384585369a8dba69b6eb3ee75a8fb623..3cb1232575e7aafc2f4b1aed64d4ca3e1569bff0 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -21,4 +21,5 @@ properties: - apple,t6020-smc - apple,t8122-smc + - apple,t8132-smc - const: apple,t8103-smc - items: diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 352251c801f5a351dd7326381b2083a9012e3444..b529acd770229090c0c9c51ff4e1e7b9d5d94ab1 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -14,4 +14,5 @@ properties: oneOf: - const: apple,t8015-nvme-ans2 + - const: apple,t8132-nvme-ans2 - items: - enum: diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 0bed9e126fdaa4ae2abeec42fd4b825d4605a022..3444b237b895efb6704157f4ed64cbdd0811f4d9 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -45,4 +45,7 @@ properties: - const: apple,t8122-pcie - const: apple,t6020-pcie + - items: + - const: apple,t8132-pcie + - const: apple,t6020-pcie - const: apple,t6020-pcie @@ -52,16 +55,32 @@ properties: reg-names: - minItems: 3 - items: - - const: config - - const: rc - - const: port0 - - const: port1 - - const: port2 - - const: port3 - - const: phy0 - - const: phy1 - - const: phy2 - - const: phy3 + oneOf: + - minItems: 3 + items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - const: phy0 + - const: phy1 + - const: phy2 + - items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - const: port3 + - const: phy0 + - const: phy1 + - const: phy2 + - const: phy3 ranges: @@ -93,11 +112,19 @@ patternProperties: properties: + compatible: + const: pciclass,0604 + reg: maxItems: 1 + power-supply: + description: Regulator that supplies power to the devices on this port + pwren-gpios: description: Optional GPIO to power on the device maxItems: 1 + interrupt-controller: true + required: - reset-gpios @@ -107,4 +134,6 @@ patternProperties: - interrupt-map + unevaluatedProperties: false + required: - compatible @@ -133,13 +162,35 @@ allOf: interrupts: maxItems: 3 + - if: + allOf: + - properties: + compatible: + contains: + const: apple,t6020-pcie + - not: + properties: + compatible: + contains: + const: apple,t8132-pcie + then: + properties: + reg-names: + minItems: 10 - if: properties: compatible: contains: - const: apple,t6020-pcie + const: apple,t8132-pcie then: properties: + reg: + minItems: 8 + maxItems: 8 reg-names: - minItems: 10 + minItems: 8 + maxItems: 8 + interrupts: + minItems: 3 + maxItems: 3 examples: diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index bc6d41b63b5d1f0a26b4457dd602e641c7485ba0..0f06ecd16143cc5fcb1876b4623837204903e017 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -89,4 +89,19 @@ patternProperties: additionalProperties: false + '-hog$': + type: object + description: GPIO hog node + properties: + gpio-hog: true + gpios: true + input: true + output-high: true + output-low: true + line-name: true + required: + - gpio-hog + - gpios + additionalProperties: false + allOf: - $ref: pinctrl.yaml# diff --git a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml index fceed47b2cd17f8e2e23839ef8dcecb2b9cf8a59..8763cd76c680c9057123d7971fa9ea1108d1996f 100644 --- a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml +++ b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -26,4 +26,5 @@ properties: - apple,t8015-spmi - apple,t8122-spmi + - apple,t8132-spmi - const: apple,t8103-spmi - items: diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index f70c33f32c5d6172224c524fe8e7218071eb4e9b..f49e1a03779e5cbab7397f57a554b35b66ee24a6 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -25,4 +25,5 @@ properties: - apple,t6020-dwc3 - apple,t8112-dwc3 + - apple,t8132-dwc3 - const: apple,t8103-dwc3 - const: apple,t8103-dwc3 diff --git a/arch/arm64/boot/dts/apple/t8132-gpu.dtsi b/arch/arm64/boot/dts/apple/t8132-gpu.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..a7cefeb2cf38681b41d643365878cbe3b8880c3a --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8132-gpu.dtsi @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* T8132 GPU resources. Firmware memory addresses are supplied by m1n1. */ + +/ { + aliases { + gpu = &gpu; + }; + + reserved-memory { + uat_ttbs: uat-ttbs { + status = "disabled"; + }; + uat_pagetables: uat-pagetables { + status = "disabled"; + }; + uat_l2: uat-l2 { + status = "disabled"; + }; + uat_handoff: uat-handoff { + status = "disabled"; + }; + gpu_firmware: gpu-firmware { + status = "disabled"; + }; + }; +}; + +&soc { + gpu: gpu@302600000 { + compatible = "apple,agx-t8132"; + reg = <0x3 0x02600000 0 0x88000>, + <0x3 0x00000000 0 0x4000000>; + reg-names = "asc", "sgx"; + mboxes = <&agx_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_l2>, + <&uat_handoff>, <&gpu_firmware>; + memory-region-names = "ttbs", "pagetables", "l2", + "handoff", "firmware"; + /* Enabled only after the loader has reserved the firmware state. */ + status = "disabled"; + }; + + agx_mbox: mailbox@302608000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x02608000 0 0x4000>; + interrupt-parent = <&aic>; + /* ADT pairs are not-empty, empty; Linux names are reversed. */ + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_gfx>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8132-j604.dts b/arch/arm64/boot/dts/apple/t8132-j604.dts index c4be915cb33ab2026da967935ed76daa3bb9d85f..4360d10ec5e23ebcf63caa7c00d04a29bc8c26ff 100644 --- a/arch/arm64/boot/dts/apple/t8132-j604.dts +++ b/arch/arm64/boot/dts/apple/t8132-j604.dts @@ -34,2 +34,83 @@ status = "okay"; }; + +/* + * Add the additional USB type C port + */ + +&nub_spmi_a1 { + hpm3: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + interrupts = <11 IRQ_TYPE_EDGE_RISING>, + <13 IRQ_TYPE_EDGE_RISING>, + <17 IRQ_TYPE_EDGE_RISING>, + <19 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "irq", "select", "sleep", "wake"; + + typec3: connector { + label = "USB-C Right"; + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec3_connector_hs: endpoint { + remote-endpoint = <&dwc3_3_hs>; + }; + }; + port@1 { + reg = <1>; + typec3_connector_ss: endpoint { + remote-endpoint = <&atcphy3_typec_lanes>; + }; + }; + }; + }; + }; +}; + +&dwc3_3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_3_hs: endpoint { + remote-endpoint = <&typec3_connector_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_3_ss: endpoint { + remote-endpoint = <&atcphy3_usb3>; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy3_typec_lanes: endpoint { + remote-endpoint = <&typec3_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy3_usb3: endpoint { + remote-endpoint = <&dwc3_3_ss>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8132-j623.dts b/arch/arm64/boot/dts/apple/t8132-j623.dts index 508e26056022c12b96bcf19b7504b92fb315764c..d8788df9a08eacb80a43d72701d76a7a702fc1c1 100644 --- a/arch/arm64/boot/dts/apple/t8132-j623.dts +++ b/arch/arm64/boot/dts/apple/t8132-j623.dts @@ -17,2 +17,14 @@ model = "Apple iMac (24-inch, 2x USB-C, M4, 2024)"; }; + +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-right"; +}; + +&typec1 { + label = "USB-C Back-left"; +}; diff --git a/arch/arm64/boot/dts/apple/t8132-j624.dts b/arch/arm64/boot/dts/apple/t8132-j624.dts index 84bdd8e343eb16c29da47428632b71054154791a..248280b6a71de40da942ae13e2fe825c6d0d23e8 100644 --- a/arch/arm64/boot/dts/apple/t8132-j624.dts +++ b/arch/arm64/boot/dts/apple/t8132-j624.dts @@ -17,2 +17,170 @@ model = "Apple iMac (24-inch, 4x USB-C, M4, 2024)"; }; + +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-right"; +}; + +&typec1 { + label = "USB-C Back-right-middle"; +}; + +/* + * Add the additional two USB type C ports + */ + +&nub_spmi_a0 { + hpm2: usb-pd@8 { + compatible = "apple,sn201202x"; + reg = <0x8 SPMI_USID>; + interrupts = <63 IRQ_TYPE_EDGE_RISING>, + <64 IRQ_TYPE_EDGE_RISING>, + <69 IRQ_TYPE_EDGE_RISING>, + <71 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "irq", "select", "sleep", "wake"; + + typec2: connector { + label = "USB-C Back-left-middle"; + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_connector_hs: endpoint { + remote-endpoint = <&dwc3_2_hs>; + }; + }; + port@1 { + reg = <1>; + typec2_connector_ss: endpoint { + remote-endpoint = <&atcphy2_typec_lanes>; + }; + }; + }; + }; + }; + + hpm3: usb-pd@6 { + compatible = "apple,sn201202x"; + reg = <0x6 SPMI_USID>; + interrupts = <89 IRQ_TYPE_EDGE_RISING>, + <91 IRQ_TYPE_EDGE_RISING>, + <95 IRQ_TYPE_EDGE_RISING>, + <97 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "irq", "select", "sleep", "wake"; + + typec3: connector { + label = "USB-C Back-left"; + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec3_connector_hs: endpoint { + remote-endpoint = <&dwc3_3_hs>; + }; + }; + port@1 { + reg = <1>; + typec3_connector_ss: endpoint { + remote-endpoint = <&atcphy3_typec_lanes>; + }; + }; + }; + }; + }; +}; + +&atcphy2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy2_typec_lanes: endpoint { + remote-endpoint = <&typec2_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy2_usb3: endpoint { + remote-endpoint = <&dwc3_2_ss>; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy3_typec_lanes: endpoint { + remote-endpoint = <&typec3_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy3_usb3: endpoint { + remote-endpoint = <&dwc3_3_ss>; + }; + }; + }; +}; + +&dwc3_2 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_2_hs: endpoint { + remote-endpoint = <&typec2_connector_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_2_ss: endpoint { + remote-endpoint = <&atcphy2_usb3>; + }; + }; + }; +}; + +&dwc3_3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_3_hs: endpoint { + remote-endpoint = <&typec3_connector_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_3_ss: endpoint { + remote-endpoint = <&atcphy3_usb3>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts index b1f779c37d5fe6dfe5a5e046ebfdd50a0aa5a35c..e1fc5d47f487669b3644db972319ec1920b95b2e 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713.dts +++ b/arch/arm64/boot/dts/apple/t8132-j713.dts @@ -16,5 +16,5 @@ / { compatible = "apple,j713", "apple,t8132", "apple,arm-platform"; - model = "Apple MacBook Air (13-inch, M4, 2024)"; + model = "Apple MacBook Air (13-inch, M4, 2025)"; led-controller { diff --git a/arch/arm64/boot/dts/apple/t8132-j773g.dts b/arch/arm64/boot/dts/apple/t8132-j773g.dts index eb2157f8e4285b1f6c3bd367b515a82ffc17dfd4..20b84f6b6a6f8cc74f7d3000fc31cfe2d39f46da 100644 --- a/arch/arm64/boot/dts/apple/t8132-j773g.dts +++ b/arch/arm64/boot/dts/apple/t8132-j773g.dts @@ -12,8 +12,26 @@ #include "t8132.dtsi" #include "t8132-jxxx.dtsi" +#include "t8132-gpu.dtsi" / { compatible = "apple,j773g", "apple,t8132", "apple,arm-platform"; model = "Apple Mac mini (M4, 2024)"; + + aliases { + ethernet0 = ðernet0; + sio = &sio; + }; +}; + +/* Persistent console capture for early T8132 bring-up failures. Keep this + * near the top of RAM, outside m1n1's kernel/initramfs load area. */ +&{/reserved-memory} { + ramoops@103e0000000 { + compatible = "ramoops"; + reg = <0x103 0xe0000000 0x0 0x200000>; + record-size = <0x40000>; + console-size = <0x100000>; + pmsg-size = <0x40000>; + }; }; @@ -24,2 +42,227 @@ power-domains = <&ps_dispext0_cpu>, <&ps_dptx_phy>; }; + +&sio_dart { + status = "okay"; +}; + +&sio_mbox { + status = "okay"; +}; + +&sio { + status = "okay"; +}; + +&dpaudio0 { + status = "okay"; +}; + +&disp0_dart { + status = "okay"; +}; + +&dcp_dart { + status = "okay"; +}; + +&dcp_mbox { + status = "okay"; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext0_mbox { + status = "okay"; +}; + +&dispext1_dart { + status = "okay"; +}; + +&dcpext1_dart { + status = "okay"; +}; + +&dcpext1_mbox { + status = "okay"; +}; + +&dptxphy { + status = "okay"; +}; + +&dcp { + status = "okay"; + /* Expose the primary and overlay surfaces. */ + apple,iomfb-surfaces = <1 1 0 0>; + apple,iomfb-surface-order = <0 1>; + apple,iomfb-primary-surface = <0>; + apple,connector-type = "HDMI-A"; + phys = <&dptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; + hdmi-pwren-gpios = <&smc_gpio 15 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; +}; + +/* The internal SSD is part of the M4 mini Linux bring-up environment. */ +&nvme { + status = "okay"; +}; + +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-middle"; +}; + +&typec1 { + label = "USB-C Back-left"; +}; + +// The front ports are on dwc/atcphy2 via a hub + +&nub_spmi_a0 { + hpm3: usb-pd@8 { + compatible = "apple,sn201202x"; + reg = <0x8 SPMI_USID>; + interrupts = <63 IRQ_TYPE_EDGE_RISING>, + <64 IRQ_TYPE_EDGE_RISING>, + <69 IRQ_TYPE_EDGE_RISING>, + <71 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "irq", "select", "sleep", "wake"; + + typec3: connector { + label = "USB-C Back-right"; + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec3_connector_hs: endpoint { + remote-endpoint = <&dwc3_3_hs>; + }; + }; + port@1 { + reg = <1>; + typec3_connector_ss: endpoint { + remote-endpoint = <&atcphy3_typec_lanes>; + }; + }; + }; + }; + }; +}; + +&atcphy3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy3_typec_lanes: endpoint { + remote-endpoint = <&typec3_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy3_usb3: endpoint { + remote-endpoint = <&dwc3_3_ss>; + }; + }; + }; +}; + +&dwc3_3 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_3_hs: endpoint { + remote-endpoint = <&typec3_connector_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_3_ss: endpoint { + remote-endpoint = <&atcphy3_usb3>; + }; + }; + }; +}; + +/* The two front ports are behind a permanently attached host-only hub. */ +&dwc3_2 { + dr_mode = "host"; + /delete-property/ role-switch-default-mode; +}; + +&atcphy2 { + apple,fixed-usb3; + apple,fixed-lane-swap; + /delete-property/ orientation-switch; + /delete-property/ mode-switch; +}; + +&pinctrl_ap { + usb3hub-flash-oe-hog { + gpio-hog; + gpios = <101 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "usb3hub-flash-oe"; + }; + + usb3hub-reset-hog { + gpio-hog; + gpios = <198 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "usb3hub-reset"; + }; +}; + +&wifi0 { + brcm,board-type = "apple,sakhalin"; +}; + +&bluetooth0 { + brcm,board-type = "apple,sakhalin"; +}; + +&port02 { + bus-range = <2 2>; + status = "okay"; + ethernet0: ethernet@0,0 { + reg = <0x20000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; +}; + +&pcie2_dart { + status = "okay"; +}; + +&pcie0 { + iommu-map = <0x100 &pcie0_dart 1 1>, + <0x200 &pcie2_dart 1 1>; +}; + +#include "hwmon-common.dtsi" +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8132-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8132-jxxx.dtsi index 2a18f5cace0906aadee28b3e7aea615a2e778e89..a5841f0ff6323c98f0bc73997ab8acab84341ec1 100644 --- a/arch/arm64/boot/dts/apple/t8132-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-jxxx.dtsi @@ -12,5 +12,18 @@ / { aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; + bluetooth0 = &bluetooth0; + dcp = &dcp; + dcpext0 = &dcpext0; + dcpext1 = &dcpext1; + disp0 = &display; + disp0_piodma = &disp0_piodma; + dispext0_piodma = &dispext0_piodma; + dispext1_piodma = &dispext1_piodma; serial0 = &serial0; + wifi0 = &wifi0; }; @@ -42,4 +55,11 @@ reg = <0x8 0 0x2 0>; /* To be filled by loader */ }; + + vreg_pcie0_port0: regulator-pcie0-port0 { + compatible = "regulator-fixed"; + regulator-name = "pwren-pcie0-port0"; + gpio = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; }; @@ -47,2 +67,185 @@ status = "okay"; }; + +/* USB */ + +/* + * All models have at least these two ports. + * Additional ports are added in t8132-j{604,624,773g}.dts respectively. + */ + +&nub_spmi_a0 { + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + interrupts = <11 IRQ_TYPE_EDGE_RISING>, + <13 IRQ_TYPE_EDGE_RISING>, + <17 IRQ_TYPE_EDGE_RISING>, + <19 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "irq", "select", "sleep", "wake"; + + typec0: connector { + label = "USB-C Left-front"; + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_connector_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_connector_ss: endpoint { + remote-endpoint = <&atcphy0_typec_lanes>; + }; + }; + }; + }; + }; + + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + interrupts = <37 IRQ_TYPE_EDGE_RISING>, + <39 IRQ_TYPE_EDGE_RISING>, + <43 IRQ_TYPE_EDGE_RISING>, + <45 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "irq", "select", "sleep", "wake"; + + typec1: connector { + label = "USB-C Left-back"; + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_connector_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + port@1 { + reg = <1>; + typec1_connector_ss: endpoint { + remote-endpoint = <&atcphy1_typec_lanes>; + }; + }; + }; + }; + }; +}; + +&dwc3_0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_connector_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb3>; + }; + }; + }; +}; + +&dwc3_1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_connector_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb3>; + }; + }; + }; +}; + +&atcphy0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy0_typec_lanes: endpoint { + remote-endpoint = <&typec0_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy0_usb3: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; +}; + +&atcphy1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + atcphy1_typec_lanes: endpoint { + remote-endpoint = <&typec1_connector_ss>; + }; + }; + + port@1 { + reg = <1>; + atcphy1_usb3: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; +}; + +/* + * Force the bus number assignments so that we can declare some of the + * on-board devices and properties that are populated by the bootloader + * (such as MAC addresses). + */ +&port00 { + power-supply = <&vreg_pcie0_port0>; + bus-range = <1 1>; + wifi0: wifi@0,0 { + compatible = "pci14e4,4434"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 00 00 00 00 00]; + apple,antenna-sku = "XX"; + }; + + bluetooth0: bluetooth@0,1 { + compatible = "pci14e4,5f72"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi index 00a09687de241f8019a2fe126c157ef61f3f098c..28395dd5d221b815b2ec6bc3c7fb46433fdbac3f 100644 --- a/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi @@ -46,4 +46,6 @@ #reset-cells = <0>; label = "pmc"; + /* DCP accesses its PMGR dashboard while handling external unplug. */ + apple,always-on; }; @@ -153,5 +155,6 @@ label = "dispext0_cpu"; power-domains = <&ps_dispext0_fe>; - apple,min-state = <4>; + /* This firmware-owned domain is locked on T8132. */ + status = "disabled"; }; @@ -786,4 +789,6 @@ label = "dispext1_cpu"; power-domains = <&ps_dispext1_fe>; + /* This firmware-owned domain is locked on T8132. */ + status = "disabled"; }; @@ -794,4 +799,6 @@ #reset-cells = <0>; label = "ans"; + /* Turning ANS off triggers a watchdog reset after a few minutes. */ + apple,always-on; }; @@ -961,5 +968,6 @@ label = "disp_cpu"; power-domains = <&ps_disp_fe>; - apple,min-state = <4>; + /* This firmware-owned domain is locked on T8132. */ + status = "disabled"; }; }; diff --git a/arch/arm64/boot/dts/apple/t8132.dtsi b/arch/arm64/boot/dts/apple/t8132.dtsi index 715980b5c7de48200f5be282b16140e91833ac19..a8b9f87e6ce9360dde83e4a44144b3096a7e253b 100644 --- a/arch/arm64/boot/dts/apple/t8132.dtsi +++ b/arch/arm64/boot/dts/apple/t8132.dtsi @@ -6,7 +6,10 @@ */ +#include #include #include #include +#include +#include / { @@ -62,7 +65,12 @@ device_type = "cpu"; reg = <0x0 0x0>; + capacity-dmips-mhz = <741>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&donan_e_opp>; + performance-domains = <&cpufreq_e>; }; @@ -71,7 +79,12 @@ device_type = "cpu"; reg = <0x0 0x1>; + capacity-dmips-mhz = <741>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&donan_e_opp>; + performance-domains = <&cpufreq_e>; }; @@ -80,7 +93,12 @@ device_type = "cpu"; reg = <0x0 0x2>; + capacity-dmips-mhz = <741>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&donan_e_opp>; + performance-domains = <&cpufreq_e>; }; @@ -89,7 +107,12 @@ device_type = "cpu"; reg = <0x0 0x3>; + capacity-dmips-mhz = <741>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&donan_e_opp>; + performance-domains = <&cpufreq_e>; }; @@ -98,7 +121,12 @@ device_type = "cpu"; reg = <0x0 0x4>; + capacity-dmips-mhz = <741>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&donan_e_opp>; + performance-domains = <&cpufreq_e>; }; @@ -107,7 +135,12 @@ device_type = "cpu"; reg = <0x0 0x5>; + capacity-dmips-mhz = <741>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&donan_e_opp>; + performance-domains = <&cpufreq_e>; }; @@ -116,7 +149,12 @@ device_type = "cpu"; reg = <0x0 0x10100>; + capacity-dmips-mhz = <1024>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&donan_p_opp>; + performance-domains = <&cpufreq_p>; }; @@ -125,7 +163,12 @@ device_type = "cpu"; reg = <0x0 0x10101>; + capacity-dmips-mhz = <1024>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&donan_p_opp>; + performance-domains = <&cpufreq_p>; }; @@ -134,7 +177,12 @@ device_type = "cpu"; reg = <0x0 0x10102>; + capacity-dmips-mhz = <1024>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&donan_p_opp>; + performance-domains = <&cpufreq_p>; }; @@ -143,7 +191,12 @@ device_type = "cpu"; reg = <0x0 0x10103>; + capacity-dmips-mhz = <1024>; enable-method = "spin-table"; cpu-release-addr = <0 0>; next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&donan_p_opp>; + performance-domains = <&cpufreq_p>; }; @@ -163,4 +216,56 @@ }; + donan_e_opp: opp-table-e { + compatible = "operating-points-v2"; + opp-shared; + + opp01 { opp-hz = /bits/ 64 <900000000>; opp-level = <1>; }; + opp02 { opp-hz = /bits/ 64 <1080000000>; opp-level = <2>; }; + opp03 { opp-hz = /bits/ 64 <1476000000>; opp-level = <3>; }; + opp04 { opp-hz = /bits/ 64 <1860000000>; opp-level = <4>; }; + opp05 { opp-hz = /bits/ 64 <2172000000>; opp-level = <5>; }; + opp06 { opp-hz = /bits/ 64 <2496000000>; opp-level = <6>; }; + opp07 { opp-hz = /bits/ 64 <2808000000>; opp-level = <7>; }; + opp08 { + opp-hz = /bits/ 64 <2892000000>; + opp-level = <8>; + turbo-mode; + }; + }; + + donan_p_opp: opp-table-p { + compatible = "operating-points-v2"; + opp-shared; + + opp01 { opp-hz = /bits/ 64 <912000000>; opp-level = <1>; }; + opp02 { opp-hz = /bits/ 64 <1260000000>; opp-level = <2>; }; + opp03 { opp-hz = /bits/ 64 <1512000000>; opp-level = <3>; }; + opp04 { opp-hz = /bits/ 64 <1800000000>; opp-level = <4>; }; + opp05 { opp-hz = /bits/ 64 <2088000000>; opp-level = <5>; }; + opp06 { opp-hz = /bits/ 64 <2352000000>; opp-level = <6>; }; + opp07 { opp-hz = /bits/ 64 <2616000000>; opp-level = <7>; }; + opp08 { opp-hz = /bits/ 64 <2868000000>; opp-level = <8>; }; + opp09 { opp-hz = /bits/ 64 <3096000000>; opp-level = <9>; }; + opp10 { opp-hz = /bits/ 64 <3300000000>; opp-level = <10>; }; + opp11 { opp-hz = /bits/ 64 <3468000000>; opp-level = <11>; }; + opp12 { opp-hz = /bits/ 64 <3624000000>; opp-level = <12>; }; + opp13 { opp-hz = /bits/ 64 <3756000000>; opp-level = <13>; }; + opp14 { opp-hz = /bits/ 64 <3876000000>; opp-level = <14>; }; + opp15 { opp-hz = /bits/ 64 <3936000000>; opp-level = <15>; }; + opp16 { opp-hz = /bits/ 64 <3984000000>; opp-level = <16>; }; + opp17 { opp-hz = /bits/ 64 <4044000000>; opp-level = <17>; }; + opp18 { opp-hz = /bits/ 64 <4104000000>; opp-level = <18>; }; + opp19 { + opp-hz = /bits/ 64 <4416000000>; + opp-level = <19>; + turbo-mode; + }; + opp20 { + opp-hz = /bits/ 64 <4464000000>; + opp-level = <20>; + turbo-mode; + }; + }; + timer { compatible = "arm,armv8-timer"; @@ -180,4 +285,25 @@ }; + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <300000000>; + clock-output-names = "disp0"; + }; + + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <300000000>; + clock-output-names = "dispext0"; + }; + + clk_dispext1: clock-dispext1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <300000000>; + clock-output-names = "dispext1"; + }; + soc: soc { compatible = "simple-bus"; @@ -187,4 +313,329 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + + cpufreq_e: cpufreq@210e20000 { + compatible = "apple,t8132-cluster-cpufreq", "apple,t8112-cluster-cpufreq"; + reg = <0x2 0x10e20000 0x0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_p: cpufreq@211e20000 { + compatible = "apple,t8132-cluster-cpufreq", "apple,t8112-cluster-cpufreq"; + reg = <0x2 0x11e20000 0x0 0x1000>; + #performance-domain-cells = <0>; + }; + + disp0_dart: iommu@30a300000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x3 0x0a300000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcp_dart: iommu@30a340000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x3 0x0a340000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcp_mbox: mailbox@30ae08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x0ae08000 0x0 0x4000>; + interrupt-parent = <&aic>; + /* The ADT lists each ASC interrupt pair as not-empty, empty. */ + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + dispext0_dart: iommu@286300000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x2 0x86300000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext0_fe>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcpext0_dart: iommu@286340000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x2 0x86340000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext0_fe>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcpext0_mbox: mailbox@286e08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x86e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + /* The ADT lists each ASC interrupt pair as not-empty, empty. */ + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext0_fe>; + status = "disabled"; + }; + + dispext1_dart: iommu@48a300000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x8a300000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext1_fe>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcpext1_dart: iommu@48a340000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x8a340000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext1_fe>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcpext1_mbox: mailbox@48ae08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x8ae08000 0x0 0x4000>; + interrupt-parent = <&aic>; + /* The ADT lists each ASC interrupt pair as not-empty, empty. */ + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext1_fe>; + status = "disabled"; + }; + + sio_dart: iommu@3ad060000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x3 0xad060000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_sio_cpu>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + sio_mbox: mailbox@3ade08000 { + compatible = "apple,t8132-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0xade08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio_cpu>; + status = "disabled"; + }; + + sio: dma-controller@3ade00000 { + compatible = "apple,t8132-sio", "apple,sio"; + reg = <0x3 0xade00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + status = "disabled"; + }; + + dpaudio0: audio-controller { + compatible = "apple,t8132-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + pmgr_dcp: power-management@382800000 { + reg = <0x3 0x82800000 0x0 0xbc000>; + reg-names = "dcp-fw-pmgr"; + #apple,bw-scratch-cells = <3>; + }; + + dcp: dcp@30ae00000 { + compatible = "apple,t8132-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x0ae00000 0x0 0x4000>, + <0x3 0x08000000 0x0 0xc20000>, + <0x3 0x0a240000 0x0 0x10000>, + <0x3 0x0a3a8000 0x0 0x4000>, + <0x3 0x0a800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x20000>; + /* Native macOS 26.6 actively uses surface 2 for video. + * Hardware composites 2 < 0 < 1 < 3 and accepts no more + * than three simultaneously active layers. */ + apple,iomfb-surfaces = <1 1 1 1>; + apple,iomfb-surface-order = <2 0 1 3>; + apple,iomfb-primary-surface = <0>; + apple,iomfb-max-layers = <3>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + status = "disabled"; + + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + dcpext0: dcp@286e00000 { + compatible = "apple,t8132-dcpext", "apple,dcpext"; + mboxes = <&dcpext0_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext0_dart 23>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x86e00000 0x0 0x4000>, + <0x2 0x84000000 0x0 0xc20000>, + <0x2 0x86240000 0x0 0x10000>, + <0x2 0x863a8000 0x0 0x4000>, + <0x2 0x86800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x2c000>; + apple,iomfb-surfaces = <1 1 0 1>; + apple,iomfb-surface-order = <0 1 3>; + apple,iomfb-primary-surface = <0>; + apple,iomfb-max-layers = <3>; + power-domains = <&ps_dispext0_fe>; + clocks = <&clk_dispext0>; + apple,dcp-index = <1>; + phandle = <&dcpext0>; + status = "disabled"; + + #address-cells = <2>; + #size-cells = <2>; + + dispext0_piodma: piodma { + iommus = <&dispext0_dart 16>; + phandle = <&dispext0_piodma>; + }; + }; + + dcpext1: dcp@48ae00000 { + compatible = "apple,t8132-dcpext", "apple,dcpext"; + mboxes = <&dcpext1_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext1_dart 23>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x4 0x8ae00000 0x0 0x4000>, + <0x4 0x88000000 0x0 0xc20000>, + <0x4 0x8a240000 0x0 0x10000>, + <0x4 0x8a3a8000 0x0 0x4000>, + <0x4 0x8a800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x34000>; + apple,iomfb-surfaces = <1 1 0 1>; + apple,iomfb-surface-order = <0 1 3>; + apple,iomfb-primary-surface = <0>; + apple,iomfb-max-layers = <3>; + power-domains = <&ps_dispext1_fe>; + clocks = <&clk_dispext1>; + apple,dcp-index = <2>; + phandle = <&dcpext1>; + status = "disabled"; + + #address-cells = <2>; + #size-cells = <2>; + + dispext1_piodma: piodma { + iommus = <&dispext1_dart 16>; + phandle = <&dispext1_piodma>; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* + * DCP scanout surfaces are real-time memory. A cacheable CPU + * alias leaves AMCC directory state behind and DCP faults when + * it subsequently reads the surface. Keep GEM DMA allocations + * write-combined on the CPU and non-cacheable at the DART. + */ + dma-noncoherent; + /* Generate a phandle explicitly for use by the loader. */ + phandle = <&display>; + }; + + dptxphy: phy@39a700000 { + compatible = "apple,t8122-dptx-phy", "apple,dptx-phy"; + reg = <0x3 0x9a700000 0x0 0x4000>, + <0x3 0x9a740000 0x0 0x10000>; + reg-names = "core", "dptx"; + power-domains = <&ps_dptx_phy>; + #phy-cells = <0>; + #reset-cells = <0>; + status = "disabled"; + }; pmgr: power-management@380700000 { @@ -343,4 +794,11 @@ ; }; + + pcie_pins: pcie-pins { + // clkreq pins + pinmux = , + , + ; + }; }; @@ -435,4 +893,442 @@ status = "disabled"; }; + + smc_mbox: mbox@38c608000 { + compatible = "apple,t8132-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x8c608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@38c600000 { + compatible = "apple,t8132-smc", "apple,t8103-smc"; + reg = <0x3 0x8c600000 0x0 0x4000>, + <0x3 0x8de00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + }; + + nub_spmi_a0: spmi@388908000 { + compatible = "apple,t8132-spmi", "apple,t8103-spmi"; + reg = <0x3 0x88908000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + #interrupt-cells = <2>; + interrupt-controller; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_nub_spmi_a0>; + }; + + nub_spmi_a1: spmi@388a08000 { + compatible = "apple,t8132-spmi", "apple,t8103-spmi"; + reg = <0x3 0x88a08000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + #interrupt-cells = <2>; + interrupt-controller; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_nub_spmi_a1>; + }; + + dwc3_0: usb@402280000 { + compatible = "apple,t8132-dwc3", "apple,t8103-dwc3"; + reg = <0x4 0x02280000 0x0 0xcd00>, <0x4 0x0228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, + <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_0_dart_0: iommu@402f00000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + }; + + dwc3_0_dart_1: iommu@402f80000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + }; + + atcphy0: phy@403000000 { + compatible = "apple,t8132-atcphy", "apple,t8122-atcphy"; + reg = <0x4 0x03000000 0x0 0x4c000>, + <0x4 0x03050000 0x0 0x8000>, + <0x4 0x00000000 0x0 0x8000>, + <0x4 0x02a90000 0x0 0x4000>, + <0x4 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_1: usb@40a280000 { + compatible = "apple,t8132-dwc3", "apple,t8103-dwc3"; + reg = <0x4 0x0a280000 0x0 0xcd00>, <0x4 0x0a28cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, + <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_1_dart_0: iommu@40af00000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x0af00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + }; + + dwc3_1_dart_1: iommu@40af80000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x0af80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + }; + + atcphy1: phy@40b000000 { + compatible = "apple,t8132-atcphy", "apple,t8122-atcphy"; + reg = <0x4 0x0b000000 0x0 0x4c000>, + <0x4 0x0b050000 0x0 0x8000>, + <0x4 0x08000000 0x0 0x8000>, + <0x4 0x0aa90000 0x0 0x4000>, + <0x4 0x0aa84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_2: usb@412280000 { + compatible = "apple,t8132-dwc3", "apple,t8103-dwc3"; + reg = <0x4 0x12280000 0x0 0xcd00>, <0x4 0x1228cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_2_dart_0 0>, + <&dwc3_2_dart_1 1>; + power-domains = <&ps_atc2_usb>; + resets = <&atcphy2>; + phys = <&atcphy2 PHY_TYPE_USB2>, <&atcphy2 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_2_dart_0: iommu@412f00000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x12f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc2_usb>; + #iommu-cells = <1>; + }; + + dwc3_2_dart_1: iommu@412f80000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x12f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc2_usb>; + #iommu-cells = <1>; + }; + + atcphy2: phy@413000000 { + compatible = "apple,t8132-atcphy", "apple,t8122-atcphy"; + reg = <0x4 0x13000000 0x0 0x4c000>, + <0x4 0x13050000 0x0 0x8000>, + <0x4 0x10000000 0x0 0x8000>, + <0x4 0x12a90000 0x0 0x4000>, + <0x4 0x12a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc2_usb>; + }; + + dwc3_3: usb@41a280000 { + compatible = "apple,t8132-dwc3", "apple,t8103-dwc3"; + reg = <0x4 0x1a280000 0x0 0xcd00>, <0x4 0x1a28cd00 0x0 0x3200>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_3_dart_0 0>, + <&dwc3_3_dart_1 1>; + power-domains = <&ps_atc3_usb>; + resets = <&atcphy3>; + phys = <&atcphy3 PHY_TYPE_USB2>, <&atcphy3 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + dwc3_3_dart_0: iommu@41af00000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x1af00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc3_usb>; + #iommu-cells = <1>; + }; + + dwc3_3_dart_1: iommu@41af80000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x1af80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_atc3_usb>; + #iommu-cells = <1>; + }; + + atcphy3: phy@41b000000 { + compatible = "apple,t8132-atcphy", "apple,t8122-atcphy"; + reg = <0x4 0x1b000000 0x0 0x4c000>, + <0x4 0x1b050000 0x0 0x8000>, + <0x4 0x18000000 0x0 0x8000>, + <0x4 0x1aa90000 0x0 0x4000>, + <0x4 0x1aa84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + power-domains = <&ps_atc3_usb>; + }; + + pcie0_dart: iommu@490000000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x90000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + }; + + pcie1_dart: iommu@491000000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x91000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + + pcie2_dart: iommu@492000000 { + compatible = "apple,t8132-dart", "apple,t8110-dart"; + reg = <0x4 0x92000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + + pcie0: pcie@1cb0000000 { + compatible = "apple,t8132-pcie", "apple,t6020-pcie"; + device_type = "pci"; + + reg = <0x1c 0xb0000000 0x0 0x1000000>, /* config */ + <0x4 0x94000000 0x0 0x4000>, /* rc */ + <0x4 0x90028000 0x0 0x4000>, /* port0 */ + <0x4 0x91028000 0x0 0x4000>, /* port1 */ + <0x4 0x92028000 0x0 0x4000>, /* port2 */ + <0x4 0x97020000 0x0 0x4000>, /* phy0 */ + <0x4 0x97024000 0x0 0x4000>, /* phy1 */ + <0x4 0x97028000 0x0 0x4000>; /* phy2 */ + reg-names = "config", "rc", "port0", "port1", "port2", "phy0", "phy1", "phy2"; + + interrupt-parent = <&aic>; + interrupts = , + , + ; + + msi-controller; + msi-parent = <&pcie0>; + msi-ranges = <&aic AIC_IRQ 1506 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x100 &pcie0_dart 0 1>, + <0x200 &pcie1_dart 1 1>, + <0x300 &pcie2_dart 1 1>; + + iommu-map-mask = <0xff00>; + + bus-range = <0 3>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0x0 0x20000000>, + <0x02000000 0x0 0x80000000 0xb 0x80000000 0x0 0x40000000>; + + power-domains = <&ps_apcie_phy_sw>; + pinctrl-0 = <&pcie_pins>; + pinctrl-names = "default"; + + port00: pci@0,0 { + compatible = "pciclass,0604"; + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 163 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + }; + + port01: pci@1,0 { + compatible = "pciclass,0604"; + device_type = "pci"; + reg = <0x800 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 164 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; + status = "disabled"; + }; + + port02: pci@2,0 { + compatible = "pciclass,0604"; + device_type = "pci"; + reg = <0x1000 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 165 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; + status = "disabled"; + }; + }; + + ans_mbox: mbox@481608000 { + compatible = "apple,t8132-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x81608000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + + #mbox-cells = <0>; + power-domains = <&ps_ans>; + }; + + sart: sart@485c50000 { + compatible = "apple,t8132-sart", "apple,t6000-sart"; + reg = <0x4 0x85c50000 0x0 0x10000>; + power-domains = <&ps_ans>; + }; + + nvme: nvme@4c5cc0000 { + compatible = "apple,t8132-nvme-ans2"; + reg = <0x4 0xc5cc0000 0x0 0x60000>, + <0x4 0x81600000 0x0 0x4000>; + reg-names = "nvme", "ans"; + + interrupt-parent = <&aic>; + interrupts = ; + + mboxes = <&ans_mbox>; + + power-domains = <&ps_ans>, <&ps_apcie_sys_st>; + power-domain-names = "ans", "apcie0"; + resets = <&ps_ans>; + + apple,sart = <&sart>; + }; }; }; diff --git a/arch/arm64/configs/m4_gpu_defconfig b/arch/arm64/configs/m4_gpu_defconfig new file mode 100644 index 0000000000000000000000000000000000000000..465ab279b22eb67a3fc0e9ff41db8867196cfc64 --- /dev/null +++ b/arch/arm64/configs/m4_gpu_defconfig @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright The Gravity Linux Contributors +CONFIG_LOCALVERSION="-m4-gpu" +CONFIG_SYSVIPC=y +CONFIG_POSIX_TIMERS=y +CONFIG_PRINTK=y +CONFIG_BUG=y +CONFIG_ELF_CORE=y +CONFIG_BASE_FULL=y +CONFIG_FUTEX=y +CONFIG_EPOLL=y +CONFIG_SIGNALFD=y +CONFIG_TIMERFD=y +CONFIG_EVENTFD=y +CONFIG_AIO=y +CONFIG_KALLSYMS=y +CONFIG_BPF_SYSCALL=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_RD_GZIP=y +CONFIG_EXPERT=y +CONFIG_MULTIUSER=y +CONFIG_RUST=y +CONFIG_ARCH_APPLE=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_47=y +CONFIG_NR_CPUS=10 +CONFIG_SCHED_MC=y +CONFIG_COMPAT=y +CONFIG_CMDLINE="console=ttySAC0,115200 earlycon=s5l,0x3ad200000 ignore_loglevel nohlt maxcpus=1" +CONFIG_PM=y +CONFIG_BINFMT_ELF=y +CONFIG_BINFMT_SCRIPT=y +CONFIG_NET=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_TTY=y +CONFIG_UNIX98_PTYS=y +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_SERIAL_SAMSUNG_UARTS=1 +CONFIG_DRM=y +CONFIG_DRM_ASAHI_M4=y +# CONFIG_DRM_ASAHI is not set +# CONFIG_APPLE_PMP is not set +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_PROC_FS=y +CONFIG_SYSFS=y +CONFIG_TMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_PRINTK_TIME=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y +CONFIG_FRAME_POINTER=y + +# Focused externally signaled dependency check in the bring-up initramfs. +CONFIG_SYNC_FILE=y +CONFIG_SW_SYNC=y diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 1b9f0cda1336dd02b8654b2af4d84f8bf04e2e5e..eedf0af4c34864eb0b664a441eb6ef816e54e9c4 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -154,4 +154,6 @@ #define APPLE_CPU_PART_M2_BLIZZARD_MAX 0x038 #define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 +#define APPLE_CPU_PART_M4_DONAN_ECORE 0x052 +#define APPLE_CPU_PART_M4_DONAN_PCORE 0x053 #define AMPERE_CPU_PART_AMPERE1 0xAC3 @@ -247,4 +249,6 @@ #define MIDR_APPLE_M2_BLIZZARD_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_BLIZZARD_MAX) #define MIDR_APPLE_M2_AVALANCHE_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_MAX) +#define MIDR_APPLE_M4_DONAN_ECORE MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M4_DONAN_ECORE) +#define MIDR_APPLE_M4_DONAN_PCORE MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M4_DONAN_PCORE) #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) #define MIDR_AMPERE1A MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1A) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 432d35b2714cd29e522ac099e23a4f8b97d305c5..af8ae705672690927e82564ed696ac66c2d18504 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2325,4 +2325,6 @@ static bool can_trap_icv_dir_el1(const struct arm64_cpu_capabilities *entry, MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_MAX), MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_MAX), + MIDR_ALL_VERSIONS(MIDR_APPLE_M4_DONAN_ECORE), + MIDR_ALL_VERSIONS(MIDR_APPLE_M4_DONAN_PCORE), {}, }; diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index bc57b290e5e7bab51a9de90d23fe36e1640e4b6b..d4e0b2c6f1e9429df2935380f9f1e2ac10641d7c 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -177,4 +177,5 @@ static const struct ftr_set_desc isar2 __prel64_initconst = { .override = &id_aa64isar2_override, .fields = { + FIELD("wfxt", ID_AA64ISAR2_EL1_WFxT_SHIFT, NULL), FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL), FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL), @@ -244,4 +245,5 @@ static const struct { "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0" }, { "arm64.nomops", "id_aa64isar2.mops=0" }, + { "arm64.nowfxt", "id_aa64isar2.wfxt=0" }, { "arm64.nomte", "id_aa64pfr1.mte=0" }, { "nokaslr", "arm64_sw.nokaslr=1" }, diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 7aa417440f6a2fd435bb6857e85fa5018db071cb..5f0408f795b93f1559203f170f128c511f4314fe 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -829,4 +829,6 @@ static const struct midr_range broken_seis[] = { MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_MAX), MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_MAX), + MIDR_ALL_VERSIONS(MIDR_APPLE_M4_DONAN_ECORE), + MIDR_ALL_VERSIONS(MIDR_APPLE_M4_DONAN_PCORE), {}, }; diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index 511f91999ed3dee63d3ed76795a72d6538d3a91c..d4a67225ea7d30cb8a6fccc6bde6e9c03b1245cf 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -112,4 +112,5 @@ struct sio_data { struct device *dev; struct apple_rtkit *rtk; + struct workqueue_struct *terminate_workqueue; void *shmem; struct sio_coproc_desc *shmem_desc_base; @@ -412,5 +413,6 @@ static int sio_terminate_all(struct dma_chan *chan) dma_cookie_complete(&siochan->current_tx->vd.tx); siochan->current_tx->terminated = true; - schedule_work(&siochan->terminate_wq); + queue_work(siochan->host->terminate_workqueue, + &siochan->terminate_wq); } vchan_get_all_descriptors(&siochan->vc, &to_free); @@ -742,8 +744,14 @@ static int sio_alloc_shmem(struct sio_data *sio) } -static int sio_send_dt_params(struct sio_data *sio) +static bool sio_param_is_deferred(u32 key) +{ + return key == 0xb || key == 0xc; +} + +static int sio_send_dt_params(struct sio_data *sio, bool deferred) { struct device_node *np = sio->dev->of_node; const char *propname = "apple,sio-firmware-params"; + bool defer_aux = of_device_is_compatible(np, "apple,t8132-sio"); int nparams, err, i; @@ -764,4 +772,11 @@ static int sio_send_dt_params(struct sio_data *sio) goto badprop; + /* Native 26.6 publishes the 0xb/0xc auxiliary region only after + * the main protocol shared memory (setup 1/2) is installed. */ + if (defer_aux && deferred != sio_param_is_deferred(key)) + continue; + if (!defer_aux && deferred) + continue; + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | FIELD_PREP(SIOMSG_PARAM, key & 0xff) | @@ -806,4 +821,10 @@ static int sio_probe(struct platform_device *pdev) sio->dev = &pdev->dev; sio->nchannels = nchannels; + sio->terminate_workqueue = devm_alloc_ordered_workqueue(&pdev->dev, + "%s-terminate", + WQ_MEM_RECLAIM, + dev_name(&pdev->dev)); + if (!sio->terminate_workqueue) + return -ENOMEM; sio->base = devm_platform_ioremap_resource(pdev, 0); @@ -874,5 +895,5 @@ static int sio_probe(struct platform_device *pdev) } - err = sio_send_dt_params(sio); + err = sio_send_dt_params(sio, false); if (err < 0) return dev_err_probe(&pdev->dev, err, "failed to send boot-up parameters\n"); @@ -882,4 +903,10 @@ static int sio_probe(struct platform_device *pdev) return err; + if (of_device_is_compatible(np, "apple,t8132-sio")) { + err = sio_send_dt_params(sio, true); + if (err < 0) + return dev_err_probe(&pdev->dev, err, + "failed to send deferred boot-up parameters\n"); + } err = dma_async_device_register(&sio->dma); if (err) diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 9d7406492360d160fd8f2a8f7cf389ea89e96af0..87ca413b194269cee36039239af78f1eac182dbc 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -222,4 +222,5 @@ obj-$(CONFIG_DRM_TVE200) += tve200/ obj-$(CONFIG_DRM_ADP) += adp/ obj-$(CONFIG_DRM_ASAHI) += asahi/ +obj-$(CONFIG_DRM_ASAHI_M4) += asahi/ obj-$(CONFIG_DRM_XEN) += xen/ obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/ diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index ecf7afd76cedaf6e69eb39e2fabbdc17cb979604..cd72ff2c8e88ff17ac249e5bb01252fbcf006220 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -22,4 +22,5 @@ config DRM_APPLE_AUDIO depends on DRM_APPLE depends on SND + select APPLE_SIO select SND_PCM select SND_DMAENGINE_PCM diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index b22839bdd611a747ae35c42165bc6ec9e1f67284..d624045a0db586040f8f653faf2d576f4d205194 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -11,4 +11,5 @@ appledrm-y += ibootep.o appledrm-y += iomfb_v12_3.o appledrm-y += iomfb_v13_3.o +appledrm-y += iomfb_v26_6.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b7f16298826925a1a4120cd3e24f72ca9533f823..7c8c4abddb6fbddee590d944f1e725e723ecd0bd 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -6,4 +6,5 @@ #include #include +#include #include #include @@ -14,4 +15,9 @@ #include "trace.h" +static bool compact_trace; +module_param_named(compact_trace, compact_trace, bool, 0644); +MODULE_PARM_DESC(compact_trace, + "Trace compact EPIC calls and reports for DCP protocol debugging"); + struct afk_receive_message_work { struct apple_dcp_afkep *ep; @@ -20,4 +26,31 @@ struct afk_receive_message_work { }; +struct afk_compact_cmd { + struct list_head node; + struct completion done; + u16 channel; + u8 type; + bool service_call; + u16 group; + u32 command; + u32 request_status; + void *output; + size_t output_len; + u32 retcode; + int status; +}; + +static u32 afk_compact_next_status(struct apple_epic_service *service) +{ + unsigned long flags; + u32 status; + + spin_lock_irqsave(&service->lock, flags); + status = (u32)service->cmd_tag++ << 8; + spin_unlock_irqrestore(&service->lock, flags); + + return status; +} + #define RBEP_TYPE GENMASK(63, 48) @@ -78,4 +111,5 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, init_completion(&afkep->started); init_completion(&afkep->stopped); + INIT_LIST_HEAD(&afkep->compact_cmds); spin_lock_init(&afkep->lock); @@ -87,6 +121,22 @@ out_free_afkep: } +void afk_cancel_commands(struct apple_dcp_afkep *afkep) +{ + struct afk_compact_cmd *cmd, *tmp; + unsigned long flags; + + spin_lock_irqsave(&afkep->lock, flags); + afkep->compact_shutting_down = true; + list_for_each_entry_safe(cmd, tmp, &afkep->compact_cmds, node) { + list_del_init(&cmd->node); + cmd->status = -ESHUTDOWN; + complete(&cmd->done); + } + spin_unlock_irqrestore(&afkep->lock, flags); +} + void afk_shutdown(struct apple_dcp_afkep *afkep) { + afk_cancel_commands(afkep); afk_send(afkep, FIELD_PREP(RBEP_TYPE, RBEP_SHUTDOWN)); int ret; @@ -139,5 +189,4 @@ static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) ep->bfr_size = size; ep->bfr_tag = tag; - reply = FIELD_PREP(RBEP_TYPE, RBEP_GETBUF_ACK); reply |= FIELD_PREP(GETBUF_ACK_DVA, ep->bfr_dma); @@ -152,4 +201,5 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, u16 tag = FIELD_GET(INITRB_TAG, message); u32 bufsz, end; + size_t block_size; if (tag != ep->bfr_tag) { @@ -182,14 +232,33 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, bfr->hdr = ep->bfr + base; bufsz = le32_to_cpu(bfr->hdr->bufsz); - if (bufsz + sizeof(*bfr->hdr) != size) { + if (bufsz > size || (size - bufsz) % 3) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: invalid ring buffer layout: size 0x%x, data 0x%x\n", + ep->endpoint, size, bufsz); + return; + } + + block_size = (size - bufsz) / 3; + if (block_size < sizeof(*bfr->hdr) || block_size % (1 << BLOCK_SHIFT)) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: ring buffer size 0x%x != expected 0x%lx\n", - ep->endpoint, bufsz, sizeof(*bfr->hdr)); + "AFK[ep:%02x]: invalid ring buffer block size 0x%lx\n", + ep->endpoint, block_size); return; } - bfr->buf = bfr->hdr + 1; + bfr->rptr = ep->bfr + base + block_size; + bfr->wptr = ep->bfr + base + 2 * block_size; + bfr->buf = ep->bfr + base + 3 * block_size; bfr->bufsz = bufsz; + bfr->block_size = block_size; bfr->ready = true; + dev_dbg(ep->dcp->dev, + "AFK[ep:%02x]: %s base=%#x size=%#x hdr=%pad rptr=%pad wptr=%pad data=%pad data-size=%#x block=%#lx\n", + ep->endpoint, bfr == &ep->txbfr ? "TX" : "RX", base, size, + &(dma_addr_t){ ep->bfr_dma + base }, + &(dma_addr_t){ ep->bfr_dma + base + block_size }, + &(dma_addr_t){ ep->bfr_dma + base + 2 * block_size }, + &(dma_addr_t){ ep->bfr_dma + base + 3 * block_size }, + bufsz, block_size); if (ep->rxbfr.ready && ep->txbfr.ready) @@ -213,4 +282,6 @@ afk_match_service(struct apple_dcp_afkep *ep, const char *name) { const struct apple_epic_service_ops *ops; + const char *service_name; + size_t len; if (!name[0]) @@ -219,6 +290,18 @@ afk_match_service(struct apple_dcp_afkep *ep, const char *name) return NULL; + service_name = strchr(name, ':'); + if (service_name) + service_name++; + else + service_name = name; + for (ops = ep->ops; ops->name[0]; ops++) { - if (strcmp(ops->name, name)) + if (!strcmp(ops->name, "*")) + return ops; + + len = strlen(ops->name); + if (strncmp(ops->name, service_name, len)) + continue; + if (service_name[len] && service_name[len] != ':') continue; @@ -239,4 +322,115 @@ static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep * } +static int afk_send_compact(struct apple_dcp_afkep *ep, u16 intf_id, u8 type, + u8 category, const void *payload, size_t payload_size, + size_t wire_size) +{ + struct epic_compact_hdr *chdr; + struct epic_compact_sub_hdr *cshdr; + struct afk_qe *qhdr, *qhdr2 = NULL; + unsigned long flags; + void *message; + size_t message_size, advance; + u32 rptr, wptr, old_wptr; + int ret = 0; + + if (payload_size > wire_size) + return -EINVAL; + if (!ep->txbfr.ready) + return -EIO; + + message_size = sizeof(*chdr) + sizeof(*cshdr) + wire_size; + message = kzalloc(message_size, GFP_KERNEL); + if (!message) + return -ENOMEM; + + chdr = message; + cshdr = message + sizeof(*chdr); + chdr->intf_id = cpu_to_le16(intf_id); + chdr->length = cpu_to_le32(message_size - + offsetof(struct epic_compact_hdr, timestamp)); + cshdr->type = type; + cshdr->category = category; + cshdr->unk = type == 0x12; + if (payload_size) + memcpy(cshdr + 1, payload, payload_size); + + spin_lock_irqsave(&ep->lock, flags); + chdr->seq = ep->qe_seq++; + + dma_rmb(); + rptr = le32_to_cpu(READ_ONCE(*ep->txbfr.rptr)); + wptr = le32_to_cpu(READ_ONCE(*ep->txbfr.wptr)); + old_wptr = wptr; + advance = ALIGN(sizeof(*qhdr) + message_size, ep->txbfr.block_size); + + if (advance > ep->txbfr.bufsz) { + ret = -EMSGSIZE; + goto out_unlock; + } + if (wptr < rptr && advance >= rptr - wptr) { + ret = -ENOMEM; + goto out_unlock; + } + if (wptr >= rptr) { + bool fits_above = advance < ep->txbfr.bufsz - wptr || + (advance == ep->txbfr.bufsz - wptr && rptr != 0); + + if (!fits_above && advance >= rptr) { + ret = -ENOMEM; + goto out_unlock; + } + } + + qhdr = ep->txbfr.buf + wptr; + qhdr->magic = cpu_to_le32(QE_MAGIC); + qhdr->size = cpu_to_le32(message_size); + qhdr->channel = cpu_to_le32(0); + qhdr->type = cpu_to_le32(0); + wptr += sizeof(*qhdr); + + if (message_size > ep->txbfr.bufsz - wptr) { + qhdr2 = ep->txbfr.buf; + memcpy(qhdr2, qhdr, sizeof(*qhdr)); + qhdr = qhdr2; + wptr = sizeof(*qhdr); + } + + memcpy(qhdr->data, message, message_size); + wptr = ALIGN(wptr + message_size, ep->txbfr.block_size); + if (wptr == ep->txbfr.bufsz) + wptr = 0; + + dma_wmb(); + WRITE_ONCE(*ep->txbfr.wptr, cpu_to_le32(wptr)); + dma_wmb(); + afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_RECV) | + ((u64)(rptr & 0xffff) << 32) | + ((u64)(wptr & 0xffff) << 16) | (old_wptr & 0xffff)); + +out_unlock: + spin_unlock_irqrestore(&ep->lock, flags); + kfree(message); + return ret; +} + +int afk_start_service(struct apple_epic_service *service) +{ + int ret; + + if (!service || !service->enabled) + return -EINVAL; + if (!service->ep->compact) + return 0; + if (service->started) + return 0; + + ret = afk_send_compact(service->ep, service->channel, 0x12, 0, + NULL, 0, 0); + if (!ret) + service->started = true; + return ret; +} + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) @@ -325,4 +519,342 @@ free: } +static void afk_recv_handle_compact_init(struct apple_dcp_afkep *ep, + u16 channel, const void *payload, + size_t payload_size) +{ + char wire_name[33], service_name[33]; + const char *name; + char *unit_suffix; + const struct apple_epic_service_ops *ops; + struct apple_epic_service *service; + s64 unit = -1; + u32 ch_idx; + int ret; + + if (payload_size < 32) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: short compact service announcement\n", + ep->endpoint); + return; + } + if (afk_epic_find_service(ep, channel)) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: duplicate compact service on interface %u\n", + ep->endpoint, channel); + return; + } + memcpy(wire_name, payload, 32); + wire_name[32] = '\0'; + name = strchr(wire_name, ':'); + if (name) + name++; + else + name = wire_name; + strscpy(service_name, name, sizeof(service_name)); + + unit_suffix = strrchr(service_name, ':'); + if (unit_suffix && !kstrtos64(unit_suffix + 1, 10, &unit)) + *unit_suffix = '\0'; + else + unit = -1; + + ops = afk_match_service(ep, wire_name); + if (!ops) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: unable to match compact service %s on interface %u\n", + ep->endpoint, wire_name, channel); + return; + } + + /* Compact AV services are republished under fresh interface IDs after + * every external-display hotplug, without a teardown message for the + * previous interface. Replace the same logical service in place so the + * fixed service table does not fill after a handful of reconnects. + * This behavior is specific to DPDEV, DPAVSERV, and AV. Other compact + * endpoints (notably DPAVCTRL) may publish several simultaneous services + * with the same name and no unit, so name/unit is not a unique key there. */ + ch_idx = ep->num_channels; + if (ep->endpoint >= DPDEV_ENDPOINT && ep->endpoint <= AV_ENDPOINT) { + for (u32 i = 0; i < ep->num_channels; i++) { + service = &ep->services[i]; + if (service->ops == ops && service->unit == unit && + !strcmp(service->name, service_name)) { + ch_idx = i; + break; + } + } + } + + if (ch_idx == ep->num_channels) { + for (u32 i = 0; i < ep->num_channels; i++) { + if (!ep->services[i].enabled) { + ch_idx = i; + break; + } + } + } + + if (ch_idx == ep->num_channels) { + if (ep->num_channels >= AFK_MAX_CHANNEL) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: too many enabled compact services\n", + ep->endpoint); + return; + } + ep->num_channels++; + } else if (ep->services[ch_idx].enabled) { + service = &ep->services[ch_idx]; + dev_info(ep->dcp->dev, + "AFK[ep:%02x]: replacing compact service %s interface %u with %u\n", + ep->endpoint, service_name, service->channel, channel); + afk_remove_service_debugfs(service); + service->torndown = true; + if (service->ops->teardown) + service->ops->teardown(service); + service->enabled = false; + } + + service = &ep->services[ch_idx]; + memset(service, 0, sizeof(*service)); + spin_lock_init(&service->lock); + service->enabled = true; + service->torndown = false; + service->ops = ops; + service->ep = ep; + service->channel = channel; + strscpy(service->name, service_name, sizeof(service->name)); + service->unit = unit; + service->cmd_tag = 0; + ops->init(service, service_name, service_name, unit); + + dev_info(ep->dcp->dev, + "AFK[ep:%02x]: new compact service %s on interface %u%s\n", + ep->endpoint, wire_name, channel, + service->defer_start ? " (deferred)" : ""); + afk_populate_service_debugfs(service); + + if (!service->defer_start) { + ret = afk_start_service(service); + if (ret) + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: failed to start compact service %u: %d\n", + ep->endpoint, channel, ret); + } +} + +static void afk_recv_handle_compact_reply(struct apple_dcp_afkep *ep, + u16 channel, u8 type, + const void *payload, + size_t payload_size) +{ + const struct epic_compact_desc *desc = payload; + const struct epic_service_call *call = NULL; + struct afk_compact_cmd *pending, *match = NULL; + unsigned long flags; + size_t copy_size; + + if (payload_size < sizeof(*desc)) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: compact reply is missing its descriptor\n", + ep->endpoint); + return; + } + + if (payload_size >= sizeof(*desc) + sizeof(*call)) { + call = (const void *)(desc + 1); + if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC) + call = NULL; + } + + spin_lock_irqsave(&ep->lock, flags); + list_for_each_entry(pending, &ep->compact_cmds, node) { + if (pending->channel != channel || pending->type != type) + continue; + if (pending->request_status != le32_to_cpu(desc->status)) + continue; + if (pending->service_call && + (!call || pending->group != le16_to_cpu(call->group) || + pending->command != le32_to_cpu(call->command))) + continue; + match = pending; + break; + } + if (!match) { + spin_unlock_irqrestore(&ep->lock, flags); + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: unexpected compact reply %02x on interface %u\n", + ep->endpoint, type, channel); + return; + } + + list_del_init(&match->node); + copy_size = min(match->output_len, payload_size - sizeof(*desc)); + if (copy_size && match->output) + memcpy(match->output, desc + 1, copy_size); + match->retcode = le32_to_cpu(desc->status); + match->status = 0; + complete(&match->done); + spin_unlock_irqrestore(&ep->lock, flags); +} + +static void afk_recv_handle_compact_call(struct apple_dcp_afkep *ep, + u16 channel, const void *payload, + size_t payload_size) +{ + const struct epic_compact_desc *request_desc = payload; + const struct epic_service_call *call; + struct epic_compact_desc *reply_desc; + struct epic_service_call *reply_call; + struct apple_epic_service *service; + void *reply; + size_t inner_size, call_size; + int ret; + + if (payload_size < sizeof(*request_desc) + sizeof(*call)) + return; + + call = payload + sizeof(*request_desc); + inner_size = payload_size - sizeof(*request_desc); + call_size = le32_to_cpu(call->data_len); + if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + inner_size < sizeof(*call) + call_size) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: malformed compact service call on interface %u\n", + ep->endpoint, channel); + return; + } + + if (compact_trace) + dev_info(ep->dcp->dev, + "AFK-COMPACT call ep=%02x intf=%u group=%u command=%u status=%#x size=%zu in=%*phN\n", + ep->endpoint, channel, le16_to_cpu(call->group), + le32_to_cpu(call->command), + le32_to_cpu(request_desc->status), call_size, + min_t(int, call_size, 64), call + 1); + + reply = kzalloc(payload_size, GFP_KERNEL); + if (!reply) + return; + reply_desc = reply; + reply_desc->status = request_desc->status; + reply_call = reply + sizeof(*reply_desc); + memcpy(reply_call, call, sizeof(*call)); + + service = afk_epic_find_service(ep, channel); + if (!service || !service->ops->call) { + ret = -ENOSYS; + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: no handler for compact service %u call %u:%u\n", + ep->endpoint, channel, le16_to_cpu(call->group), + le32_to_cpu(call->command)); + } else { + ret = service->ops->call(service, le16_to_cpu(call->group), + le32_to_cpu(call->command), + call + 1, call_size, reply_call + 1, + call_size); + } + if (ret) + reply_desc->status = cpu_to_le32(ret); + if (compact_trace) + dev_info(ep->dcp->dev, + "AFK-COMPACT reply ep=%02x intf=%u group=%u command=%u ret=%d status=%#x size=%zu out=%*phN\n", + ep->endpoint, channel, le16_to_cpu(call->group), + le32_to_cpu(call->command), ret, + le32_to_cpu(reply_desc->status), call_size, + min_t(int, call_size, 64), reply_call + 1); + + ret = afk_send_compact(ep, channel, EPIC_SUBTYPE_STD_SERVICE, 2, + reply, payload_size, payload_size); + if (ret) + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: failed to reply to compact service call: %d\n", + ep->endpoint, ret); + kfree(reply); +} + +static bool afk_recv_handle_compact(struct apple_dcp_afkep *ep, u32 channel, + u32 type, const void *data, + size_t data_size) +{ + const struct epic_compact_hdr *hdr = data; + const struct epic_compact_sub_hdr *sub; + struct apple_epic_service *service; + const void *payload; + size_t payload_size; + u16 intf_id; + + if (channel || type || + data_size < sizeof(*hdr) + sizeof(*sub)) + return false; + sub = data + sizeof(*hdr); + if (le32_to_cpu(hdr->length) != + data_size - offsetof(struct epic_compact_hdr, timestamp)) + return false; + + ep->compact = true; + intf_id = le16_to_cpu(hdr->intf_id); + payload = sub + 1; + payload_size = data_size - sizeof(*hdr) - sizeof(*sub); + + if (sub->category == 0 && sub->type == 0x11) { + afk_recv_handle_compact_init(ep, intf_id, payload, payload_size); + return true; + } + if (sub->category == 1 && sub->type == EPIC_SUBTYPE_STD_SERVICE) { + afk_recv_handle_compact_call(ep, intf_id, payload, payload_size); + return true; + } + if (sub->category == 2) { + afk_recv_handle_compact_reply(ep, intf_id, sub->type, payload, + payload_size); + return true; + } + if (sub->category == 0 && sub->type == EPIC_SUBTYPE_STD_SERVICE) { + service = afk_epic_find_service(ep, intf_id); + if (payload_size >= sizeof(struct epic_compact_desc)) { + payload += sizeof(struct epic_compact_desc); + payload_size -= sizeof(struct epic_compact_desc); + } + if (compact_trace) { + dev_info(ep->dcp->dev, + "AFK-COMPACT report ep=%02x intf=%u service=%s size=%zu data=%*phN\n", + ep->endpoint, intf_id, + service ? service->ops->name : "", + payload_size, min_t(int, payload_size, 64), + payload); + if (payload_size > 64) + dev_info(ep->dcp->dev, + "AFK-COMPACT report-tail ep=%02x intf=%u data=%*phN\n", + ep->endpoint, intf_id, + min_t(int, payload_size - 64, 64), + payload + 64); + } + if (service && service->ops->report) { + service->ops->report(service, EPIC_SUBTYPE_STD_SERVICE, + payload, payload_size); + } + return true; + } + if (sub->category == 0) { + service = afk_epic_find_service(ep, intf_id); + if (compact_trace) + dev_info(ep->dcp->dev, + "AFK-COMPACT notification ep=%02x intf=%u service=%s type=%02x size=%zu data=%*phN\n", + ep->endpoint, intf_id, + service ? service->ops->name : "", + sub->type, payload_size, + min_t(int, payload_size, 64), payload); + if (service && service->ops->report) + service->ops->report(service, sub->type, payload, + payload_size); + return true; + } + + dev_dbg(ep->dcp->dev, + "AFK[ep:%02x]: unhandled compact message %02x:%02x on interface %u\n", + ep->endpoint, sub->category, sub->type, intf_id); + return true; +} + static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) { @@ -478,5 +1010,5 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, return; - ret = service->ops->call(service, le32_to_cpu(call->type), + ret = service->ops->call(service, 0, le32_to_cpu(call->type), payload + sizeof(*call), call_size, reply + sizeof(*call), call_size); @@ -513,11 +1045,15 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, { struct apple_epic_service *service; - struct epic_hdr *ehdr = (struct epic_hdr *)data; - struct epic_sub_hdr *eshdr = - (struct epic_sub_hdr *)(data + sizeof(*ehdr)); - u16 subtype = le16_to_cpu(eshdr->type); - u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); + struct epic_hdr *ehdr; + struct epic_sub_hdr *eshdr; + u16 subtype; + u8 *payload; size_t payload_size; + if (afk_recv_handle_compact(ep, channel, type, data, data_size)) + return; + + ehdr = (struct epic_hdr *)data; + eshdr = (struct epic_sub_hdr *)(data + sizeof(*ehdr)); if (data_size < sizeof(*ehdr) + sizeof(*eshdr)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", @@ -525,4 +1061,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, return; } + subtype = le16_to_cpu(eshdr->type); + payload = data + sizeof(*ehdr) + sizeof(*eshdr); payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); @@ -597,6 +1135,6 @@ static bool afk_recv(struct apple_dcp_afkep *ep) } - rptr = le32_to_cpu(ep->rxbfr.hdr->rptr); - wptr = le32_to_cpu(ep->rxbfr.hdr->wptr); + rptr = le32_to_cpu(READ_ONCE(*ep->rxbfr.rptr)); + wptr = le32_to_cpu(READ_ONCE(*ep->rxbfr.wptr)); trace_afk_recv_rwptr_pre(ep, rptr, wptr); @@ -643,5 +1181,5 @@ static bool afk_recv(struct apple_dcp_afkep *ep) } - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + WRITE_ONCE(*ep->rxbfr.rptr, cpu_to_le32(rptr)); } @@ -656,5 +1194,5 @@ static bool afk_recv(struct apple_dcp_afkep *ep) type = le32_to_cpu(hdr->type); - rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); + rptr = ALIGN(rptr + sizeof(*hdr) + size, ep->rxbfr.block_size); if (WARN_ON(rptr > ep->rxbfr.bufsz)) rptr = 0; @@ -664,5 +1202,5 @@ static bool afk_recv(struct apple_dcp_afkep *ep) dma_mb(); - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + WRITE_ONCE(*ep->rxbfr.rptr, cpu_to_le32(rptr)); trace_afk_recv_rwptr_post(ep, rptr, wptr); @@ -756,6 +1294,6 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, dma_rmb(); - rptr = le32_to_cpu(ep->txbfr.hdr->rptr); - wptr = le32_to_cpu(ep->txbfr.hdr->wptr); + rptr = le32_to_cpu(READ_ONCE(*ep->txbfr.rptr)); + wptr = le32_to_cpu(READ_ONCE(*ep->txbfr.wptr)); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; @@ -870,10 +1408,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; - wptr = ALIGN(wptr, 1 << BLOCK_SHIFT); + wptr = ALIGN(wptr, ep->txbfr.block_size); if (wptr == ep->txbfr.bufsz) wptr = 0; trace_afk_send_rwptr_post(ep, rptr, wptr); - ep->txbfr.hdr->wptr = cpu_to_le32(wptr); + WRITE_ONCE(*ep->txbfr.wptr, cpu_to_le32(wptr)); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); @@ -885,4 +1423,83 @@ out: } +static int afk_send_compact_command(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, + u32 request_status, u32 *retcode, + unsigned long timeout) +{ + struct apple_dcp_afkep *ep = service->ep; + const struct epic_service_call *call = payload; + struct afk_compact_cmd pending; + struct epic_compact_desc *desc; + unsigned long flags; + void *wrapped; + size_t wire_size = max(payload_len, output_len); + unsigned long waited; + int ret; + + wrapped = kzalloc(sizeof(*desc) + wire_size, GFP_KERNEL); + if (!wrapped) + return -ENOMEM; + desc = wrapped; + desc->status = cpu_to_le32(request_status); + desc->length = cpu_to_le32(wire_size); + if (payload_len) + memcpy(desc + 1, payload, payload_len); + + memset(&pending, 0, sizeof(pending)); + INIT_LIST_HEAD(&pending.node); + init_completion(&pending.done); + pending.channel = service->channel; + pending.type = type; + pending.request_status = request_status; + pending.output = output; + pending.output_len = output_len; + pending.status = -EINPROGRESS; + if (payload_len >= sizeof(*call) && + le32_to_cpu(call->magic) == EPIC_SERVICE_CALL_MAGIC) { + pending.service_call = true; + pending.group = le16_to_cpu(call->group); + pending.command = le32_to_cpu(call->command); + } + + spin_lock_irqsave(&ep->lock, flags); + if (ep->compact_shutting_down) { + spin_unlock_irqrestore(&ep->lock, flags); + ret = -ESHUTDOWN; + goto out; + } + list_add_tail(&pending.node, &ep->compact_cmds); + spin_unlock_irqrestore(&ep->lock, flags); + + ret = afk_send_compact(ep, service->channel, type, 1, wrapped, + sizeof(*desc) + payload_len, + sizeof(*desc) + wire_size); + if (ret) + goto remove_pending; + + waited = wait_for_completion_timeout(&pending.done, timeout); + spin_lock_irqsave(&ep->lock, flags); + if (!waited && !list_empty(&pending.node)) { + list_del_init(&pending.node); + pending.status = -ETIMEDOUT; + } + ret = pending.status; + if (retcode) + *retcode = pending.retcode; + spin_unlock_irqrestore(&ep->lock, flags); + goto out; + +remove_pending: + spin_lock_irqsave(&ep->lock, flags); + if (!list_empty(&pending.node)) + list_del_init(&pending.node); + pending.status = ret; + spin_unlock_irqrestore(&ep->lock, flags); +out: + kfree(wrapped); + return ret; +} + int afk_send_command(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, @@ -898,4 +1515,10 @@ int afk_send_command(struct apple_epic_service *service, u8 type, DECLARE_COMPLETION_ONSTACK(completion); + if (ep->compact) + return afk_send_compact_command(service, type, payload, + payload_len, output, output_len, + afk_compact_next_status(service), + retcode, msecs_to_jiffies(5000)); + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); @@ -985,7 +1608,9 @@ err_free_rxbuf: } -int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, - const void *data, size_t data_len, size_t data_pad, - void *output, size_t output_len, size_t output_pad) +static int __afk_service_call(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, u32 *wire_status, + unsigned long timeout) { struct epic_service_call *call; @@ -994,5 +1619,5 @@ int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, sizeof(*call); int ret; - u32 retcode; + u32 request_status = 0, retcode = 0; u32 retlen; @@ -1011,9 +1636,24 @@ int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, memcpy(bfr + sizeof(*call), data, data_len); - ret = afk_send_command(service, EPIC_SUBTYPE_STD_SERVICE, bfr, bfr_len, - bfr, bfr_len, &retcode); + if (service->ep->compact) { + request_status = wire_status ? *wire_status : + afk_compact_next_status(service); + ret = afk_send_compact_command(service, + EPIC_SUBTYPE_STD_SERVICE, + bfr, bfr_len, bfr, bfr_len, + request_status, &retcode, timeout); + } else { + ret = afk_send_command(service, EPIC_SUBTYPE_STD_SERVICE, bfr, + bfr_len, bfr, bfr_len, &retcode); + } if (ret) goto out; - if (retcode) { + if (wire_status) + *wire_status = retcode; + if (service->ep->compact && retcode != request_status) { + ret = -EIO; + goto out; + } + if (!service->ep->compact && retcode) { ret = -EINVAL; goto out; @@ -1039,4 +1679,27 @@ out: } +int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t data_len, size_t data_pad, + void *output, size_t output_len, size_t output_pad) +{ + return __afk_service_call(service, group, command, data, data_len, + data_pad, output, output_len, output_pad, + NULL, msecs_to_jiffies(5000)); +} + +int afk_service_call_status(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, u32 *status, + unsigned long timeout) +{ + if (!service->ep->compact) + return -EOPNOTSUPP; + + return __afk_service_call(service, group, command, data, data_len, + data_pad, output, output_len, output_pad, + status, timeout); +} + #if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) @@ -1161,5 +1824,6 @@ static void afk_populate_service_debugfs(struct apple_epic_service *srv) return; - if (strcmp(srv->ops->name, "DCPAVAudioInterface") == 0) { + if (!strcmp(srv->ops->name, "DCPAVAudioInterface") || + !strcmp(srv->ops->name, "dcpav-audio-interface-")) { srv->debugfs.entry = debugfs_create_dir(srv->ops->name, srv->ep->debugfs_entry); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index a339c00a2a0138ea981bfbeeb0e41a2357682541..23739ca5197db08c6b7f70936ef4a33827312c10 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -10,4 +10,5 @@ #include #include +#include #include @@ -39,4 +40,6 @@ struct apple_epic_service { const struct apple_epic_service_ops *ops; struct apple_dcp_afkep *ep; + char name[33]; + s64 unit; struct epic_cmd_info cmds[MAX_PENDING_CMDS]; @@ -48,4 +51,6 @@ struct apple_epic_service { bool enabled; bool torndown; + bool defer_start; + bool started; void *cookie; @@ -64,5 +69,5 @@ struct apple_epic_service_ops { void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); - int (*call)(struct apple_epic_service *service, u32 idx, + int (*call)(struct apple_epic_service *service, u16 group, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); @@ -74,10 +79,5 @@ struct apple_epic_service_ops { struct afk_ringbuffer_header { __le32 bufsz; - u32 unk; - u32 _pad1[14]; - __le32 rptr; - u32 _pad2[15]; - __le32 wptr; - u32 _pad3[15]; + __le32 unk; }; @@ -131,4 +131,25 @@ struct epic_service_call { static_assert(sizeof(struct epic_service_call) == 64); +struct epic_compact_hdr { + u8 seq; + u8 flags; + __le16 intf_id; + __le32 length; + __le64 timestamp; +} __packed; + +struct epic_compact_sub_hdr { + u8 type; + u8 category; + u8 unk; + u8 flags; + __le32 reserved; +} __packed; + +struct epic_compact_desc { + __le32 status; + __le32 length; +} __packed; + enum epic_type { EPIC_TYPE_NOTIFY = 0, @@ -146,4 +167,5 @@ enum epic_category { enum epic_subtype { + EPIC_SUBTYPE_NOTIFY = 0x14, EPIC_SUBTYPE_ANNOUNCE = 0x30, EPIC_SUBTYPE_TEARDOWN = 0x32, @@ -154,7 +176,9 @@ struct afk_ringbuffer { bool ready; struct afk_ringbuffer_header *hdr; - u32 rptr; + __le32 *rptr; + __le32 *wptr; void *buf; size_t bufsz; + size_t block_size; }; @@ -178,4 +202,8 @@ struct apple_dcp_afkep { spinlock_t lock; u16 qe_seq; + bool compact; + + struct list_head compact_cmds; + bool compact_shutting_down; const struct apple_epic_service_ops *ops; @@ -191,4 +219,6 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops); int afk_start(struct apple_dcp_afkep *ep); +int afk_start_service(struct apple_epic_service *service); +void afk_cancel_commands(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); @@ -202,3 +232,8 @@ int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad); +int afk_service_call_status(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, u32 *status, + unsigned long timeout); #endif diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 17b59033f12dae5337c5efb32b1cff3cad1371c4..005afc281a1949d50e784b1728ecc04b15925877 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -11,6 +11,8 @@ #include #include +#include #include #include +#include #include #include @@ -18,4 +20,10 @@ #include #include +#include +#include +#include + +#include +#include #include @@ -35,4 +43,5 @@ #include #include +#include #include #include @@ -49,23 +58,256 @@ struct apple_drm_private { struct drm_device drm; + struct resource scanout_pool; + bool scanout_pool_attached; +}; + +struct apple_drm_gem_object { + struct drm_gem_dma_object dma; + struct sg_table *scanout_sgt; + dma_addr_t pool_dma_addr; + bool pool_fixed_iova; }; +static void apple_drm_gem_restore_direct_map(struct apple_drm_gem_object *obj) +{ + struct scatterlist *sg; + unsigned int i; + + if (!obj->scanout_sgt) + return; + + for_each_sg(obj->scanout_sgt->sgl, sg, + obj->scanout_sgt->orig_nents, i) { + struct page *page = sg_page(sg); + unsigned int nr_pages = PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; + unsigned int j; + + for (j = 0; j < nr_pages; j++) + set_direct_map_default_noflush(page + j); + } + + flush_tlb_all(); + sg_free_table(obj->scanout_sgt); + kfree(obj->scanout_sgt); + obj->scanout_sgt = NULL; +} + +static void apple_drm_gem_free(struct drm_gem_object *gem_obj) +{ + struct apple_drm_gem_object *obj = + container_of(to_drm_gem_dma_obj(gem_obj), + struct apple_drm_gem_object, dma); + + if (obj->pool_fixed_iova) { + obj->dma.dma_addr = obj->pool_dma_addr; + obj->pool_fixed_iova = false; + } + + apple_drm_gem_restore_direct_map(obj); + drm_gem_dma_free(&obj->dma); +} + +static void apple_drm_gem_print_info(struct drm_printer *p, + unsigned int indent, + const struct drm_gem_object *gem_obj) +{ + const struct apple_drm_gem_object *obj = + container_of(to_drm_gem_dma_obj(gem_obj), + struct apple_drm_gem_object, dma); + phys_addr_t run_start = 0, run_end = 0; + struct scatterlist *sg; + unsigned int i; + + drm_gem_dma_object_print_info(p, indent, gem_obj); + if (!obj->scanout_sgt) + return; + + drm_printf_indent(p, indent, "scanout physical backing:\n"); + for_each_sg(obj->scanout_sgt->sgl, sg, + obj->scanout_sgt->orig_nents, i) { + phys_addr_t start = sg_phys(sg); + phys_addr_t end = start + sg->length; + + if (!run_end) { + run_start = start; + run_end = end; + continue; + } + if (start == run_end) { + run_end = end; + continue; + } + + drm_printf_indent(p, indent + 1, "%pa..%pa (%pa bytes)\n", + &run_start, &run_end, + &(phys_addr_t){ run_end - run_start }); + run_start = start; + run_end = end; + } + if (run_end) + drm_printf_indent(p, indent + 1, "%pa..%pa (%pa bytes)\n", + &run_start, &run_end, + &(phys_addr_t){ run_end - run_start }); +} + +static const struct drm_gem_object_funcs apple_drm_gem_funcs = { + .free = apple_drm_gem_free, + .print_info = apple_drm_gem_print_info, + .get_sg_table = drm_gem_dma_object_get_sg_table, + .vmap = drm_gem_dma_object_vmap, + .mmap = drm_gem_dma_object_mmap, + .vm_ops = &drm_gem_dma_vm_ops, +}; + +static struct drm_gem_object * +apple_drm_gem_create_object(struct drm_device *drm, size_t size) +{ + struct apple_drm_gem_object *obj; + + obj = kzalloc_obj(*obj); + if (!obj) + return ERR_PTR(-ENOMEM); + + obj->dma.base.funcs = &apple_drm_gem_funcs; + return &obj->dma.base; +} + +static int apple_drm_gem_remove_direct_map(struct drm_gem_dma_object *dma_obj) +{ + struct apple_drm_gem_object *obj = + container_of(dma_obj, struct apple_drm_gem_object, dma); + struct scatterlist *sg; + unsigned int i; + int ret = 0; + + if (!can_set_direct_map()) + return -EOPNOTSUPP; + + obj->scanout_sgt = drm_gem_dma_get_sg_table(dma_obj); + if (IS_ERR(obj->scanout_sgt)) { + ret = PTR_ERR(obj->scanout_sgt); + obj->scanout_sgt = NULL; + return ret; + } + + for_each_sg(obj->scanout_sgt->sgl, sg, + obj->scanout_sgt->orig_nents, i) { + struct page *page = sg_page(sg); + unsigned int nr_pages = PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; + unsigned int j; + + for (j = 0; j < nr_pages; j++) { + unsigned long start = (unsigned long)page_address(page + j); + + /* dma_alloc_wc() cleans the allocator's WB alias, but a clean + * can leave an AMCC cache-directory entry behind. DCP scanout + * is realtime, so retire the entry before removing that alias. */ + dcache_clean_inval_poc(start, start + PAGE_SIZE); + ret = set_direct_map_invalid_noflush(page + j); + if (ret) + goto err_restore; + } + } + + flush_tlb_all(); + return 0; + +err_restore: + apple_drm_gem_restore_direct_map(obj); + return ret; +} + +static int apple_drm_gem_map_nomap_pool(struct drm_gem_dma_object *dma_obj) +{ + struct apple_drm_gem_object *obj = + container_of(dma_obj, struct apple_drm_gem_object, dma); + struct apple_drm_private *apple = + container_of(dma_obj->base.dev, struct apple_drm_private, drm); + struct device *dev = dma_obj->base.dev->dev; + dma_addr_t pool_dma = dma_obj->dma_addr; + phys_addr_t phys = dma_to_phys(dev, pool_dma); + dma_addr_t iova; + u64 iova_base; + resource_size_t size = dma_obj->base.size; + + if (!apple->scanout_pool_attached || + phys < apple->scanout_pool.start || + size > resource_size(&apple->scanout_pool) || + phys - apple->scanout_pool.start > + resource_size(&apple->scanout_pool) - size) { + dev_err(dev, + "scanout allocation %pa+%pa is outside no-map pool %pr\n", + &phys, &size, &apple->scanout_pool); + return -ERANGE; + } + + if (of_property_read_u64(dev->of_node, "apple,scanout-iova-base", + &iova_base)) + return -EINVAL; + + /* m1n1 installs this entire mapping before Linux and leaves the locked + * realtime DART hierarchy untouched. Selecting a buffer is therefore an + * offset calculation, not a Linux page-table update. */ + iova = iova_base + (phys - apple->scanout_pool.start); + + obj->pool_dma_addr = pool_dma; + obj->pool_fixed_iova = true; + dma_obj->dma_addr = iova; + dev_info(dev, "pre-mapped no-map scanout buffer: phys %pa -> iova %pad (%zu bytes)\n", + &phys, &iova, dma_obj->base.size); + return 0; +} + DEFINE_DRM_GEM_DMA_FOPS(apple_fops); #define DART_PAGE_SIZE 16384 +static struct drm_gem_dma_object * +apple_drm_gem_create_scanout(struct drm_device *drm, size_t size) +{ + struct drm_gem_dma_object *dma_obj; + int ret; + + dma_obj = drm_gem_dma_create(drm, round_up(size, DART_PAGE_SIZE)); + if (IS_ERR(dma_obj)) + return dma_obj; + + if (of_property_read_bool(drm->dev->of_node, + "apple,scanout-memory-is-nomap")) + ret = apple_drm_gem_map_nomap_pool(dma_obj); + else + ret = apple_drm_gem_remove_direct_map(dma_obj); + if (ret) { + drm_gem_object_put(&dma_obj->base); + return ERR_PTR(ret); + } + + return dma_obj; +} + static int apple_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm, struct drm_mode_create_dumb *args) { + struct drm_gem_dma_object *dma_obj; + int ret; + args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), 64); args->size = round_up(args->pitch * args->height, DART_PAGE_SIZE); - return drm_gem_dma_dumb_create_internal(file_priv, drm, args); + dma_obj = apple_drm_gem_create_scanout(drm, args->size); + if (IS_ERR(dma_obj)) + return PTR_ERR(dma_obj); + + ret = drm_gem_handle_create(file_priv, &dma_obj->base, &args->handle); + + drm_gem_object_put(&dma_obj->base); + return ret; } static const struct drm_driver apple_drm_driver = { - DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), + DRM_GEM_DMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), DRM_FBDEV_DMA_DRIVER_OPS, + .gem_create_object = apple_drm_gem_create_object, .name = DRIVER_NAME, .desc = DRIVER_DESC, @@ -273,7 +515,13 @@ static int apple_probe_per_dcp(struct device *dev, struct apple_connector *connector; struct apple_encoder *enc; + struct drm_plane *primary_plane = NULL; struct drm_plane *planes[DCP_MAX_PLANES]; unsigned long *iomfb_surfaces = dcp_get_iomfb_surfaces(dcp); + u32 surface_order[DCP_MAX_PLANES]; + DECLARE_BITMAP(ordered_surfaces, DCP_MAX_PLANES); + int surface_count; + int order_count; int ret; + u32 primary_surface; u32 surf; int zpos = 0; @@ -281,8 +529,50 @@ static int apple_probe_per_dcp(struct device *dev, enum drm_plane_type plane_type; - for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { - plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; + surface_count = bitmap_weight(iomfb_surfaces, DCP_MAX_PLANES); + order_count = of_property_count_u32_elems(dcp->dev.of_node, + "apple,iomfb-surface-order"); + if (order_count >= 0) { + if (order_count != surface_count) { + dev_err(dev, "iomfb surface order has %d entries, expected %d\n", + order_count, surface_count); + return -EINVAL; + } + ret = of_property_read_u32_array(dcp->dev.of_node, + "apple,iomfb-surface-order", + surface_order, surface_count); + if (ret) + return ret; + } else { + for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) + surface_order[zpos++] = surf; + } + primary_surface = surface_order[0]; + of_property_read_u32(dcp->dev.of_node, "apple,iomfb-primary-surface", + &primary_surface); + if (primary_surface >= DCP_MAX_PLANES || + !test_bit(primary_surface, iomfb_surfaces)) { + dev_err(dev, "invalid primary iomfb surface %u\n", + primary_surface); + return -EINVAL; + } + + bitmap_zero(ordered_surfaces, DCP_MAX_PLANES); + for (zpos = 0; zpos < surface_count; zpos++) { + surf = surface_order[zpos]; + if (surf >= DCP_MAX_PLANES || + !test_bit(surf, iomfb_surfaces) || + test_and_set_bit(surf, ordered_surfaces)) { + dev_err(dev, "invalid iomfb surface %u at zpos %d\n", + surf, zpos); + return -EINVAL; + } + + if (surf == primary_surface) + plane_type = DRM_PLANE_TYPE_PRIMARY; + else + plane_type = DRM_PLANE_TYPE_OVERLAY; planes[zpos] = apple_plane_init(drm, 1U << num, surf, - supports_l10r, plane_type); + supports_l10r, + plane_type); if (IS_ERR(planes[zpos])) return PTR_ERR(planes[zpos]); @@ -291,16 +581,11 @@ static int apple_probe_per_dcp(struct device *dev, if (ret) return ret; - - zpos++; + if (plane_type == DRM_PLANE_TYPE_PRIMARY) + primary_plane = planes[zpos]; } - /* - * Even though we have an overlay plane, we cannot expose it to legacy - * userspace for cursors as we cannot make the same guarantees as ye olde - * hardware cursor planes such userspace would expect us to. Modern userspace - * knows what to do with overlays. - */ crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); - ret = drm_crtc_init_with_planes(drm, &crtc->base, planes[0], NULL, + ret = drm_crtc_init_with_planes(drm, &crtc->base, primary_plane, + NULL, &apple_crtc_funcs, NULL); if (ret) @@ -374,4 +659,33 @@ err: } +static int apple_get_scanout_pool_resource(struct device *dev, + struct resource *pool_r) +{ + struct device_node *node; + int idx, ret = -ENODEV; + + idx = of_property_match_string(dev->of_node, "memory-region-names", + "scanout-pool"); + if (idx < 0) + return idx; + + node = of_parse_phandle(dev->of_node, "memory-region", idx); + if (!node) + return -ENODEV; + + if (!of_device_is_available(node) || + !of_device_is_compatible(node, "shared-dma-pool") || + !of_property_read_bool(node, "no-map")) { + dev_err(dev, "scanout-pool must be an available no-map shared-dma-pool\n"); + goto out_put; + } + + ret = of_address_to_resource(node, 0, pool_r); + +out_put: + of_node_put(node); + return ret; +} + static const struct of_device_id apple_dcp_id_tbl[] = { { .compatible = "apple,dcp" }, @@ -409,9 +723,12 @@ static int apple_drm_init_dcp(struct device *dev) continue; - ret = dcp_start(dcp[num_dcp]); + /* apple_probe_per_dcp() has permanently allocated a CRTC and its + * planes. Count it even if firmware startup fails so later DCPs get + * the CRTC index used by their possible_crtcs masks, and retain the + * matching platform device in the readiness array. */ + num_dcp++; + ret = dcp_start(dcp[num_dcp - 1]); if (ret) continue; - - num_dcp++; } @@ -448,4 +765,5 @@ static int apple_drm_init(struct device *dev) struct resource fb_r; resource_size_t fb_size; + bool use_nomap_pool; int ret; @@ -464,8 +782,27 @@ static int apple_drm_init(struct device *dev) dev_set_drvdata(dev, apple); + use_nomap_pool = of_property_read_bool(dev->of_node, + "apple,scanout-memory-is-nomap"); + if (use_nomap_pool) { + ret = apple_get_scanout_pool_resource(dev, &apple->scanout_pool); + if (ret) { + dev_err(dev, "invalid scanout-pool reserved memory: %d\n", ret); + return ret; + } + + ret = of_reserved_mem_device_init_by_name(dev, dev->of_node, + "scanout-pool"); + if (ret) { + dev_err(dev, "failed to attach scanout-pool: %d\n", ret); + return ret; + } + apple->scanout_pool_attached = true; + dev_info(dev, "using no-map scanout pool %pr\n", + &apple->scanout_pool); + } ret = component_bind_all(dev, apple); if (ret) - return ret; + goto err_release_rmem; ret = drmm_mode_config_init(&apple->drm); @@ -473,8 +810,5 @@ static int apple_drm_init(struct device *dev) goto err_unbind; - /* - * IOMFB::UPPipeDCP_H13P::verify_surfaces produces the error "plane - * requires a minimum of 32x32 for the source buffer" if smaller - */ + /* DCP requires source buffers to be at least 32x32 pixels. */ apple->drm.mode_config.min_width = 32; apple->drm.mode_config.min_height = 32; @@ -517,4 +851,9 @@ static int apple_drm_init(struct device *dev) err_unbind: component_unbind_all(dev, NULL); +err_release_rmem: + if (apple->scanout_pool_attached) { + of_reserved_mem_device_release(dev); + apple->scanout_pool_attached = false; + } return ret; } @@ -528,4 +867,8 @@ static void apple_drm_uninit(struct device *dev) component_unbind_all(dev, NULL); + if (apple->scanout_pool_attached) { + of_reserved_mem_device_release(dev); + apple->scanout_pool_attached = false; + } dev_set_drvdata(dev, NULL); diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 38718e2f56117b9e079adee28c8471a25cbdd867..656a2599a2d8d278b57f0fedac7d2082ad9572c3 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -40,8 +40,11 @@ struct dcp_audio { struct device_link *dma_link; struct dma_chan *chan; + bool dma_buffer_configured; struct snd_card *card; struct snd_jack *jack; struct snd_pcm_substream *substream; unsigned int open_cookie; + bool power_held; + bool link_started; struct mutex data_lock; @@ -60,5 +63,5 @@ struct dcp_audio { static const struct snd_pcm_hardware dcp_pcm_hw = { .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED, + SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BATCH, .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_LE | SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE, @@ -118,4 +121,5 @@ static void dcpaud_fill_fmt_sieve(struct snd_pcm_hw_params *params, int i; + memset(sieve, 0, sizeof(*sieve)); sieve->nchans = GENMASK(c->max, c->min); sieve->formats = f->bits[0] | ((u64) f->bits[1]) << 32; /* TODO: don't open-code */ @@ -214,7 +218,10 @@ static int dcpaud_rule_rate(struct snd_pcm_hw_params *params, } -static int dcpaud_init_dma(struct dcp_audio *dcpaud) +extern bool hdmi_audio; + +static int dcpaud_request_dma(struct dcp_audio *dcpaud) { struct dma_chan *chan; + if (dcpaud->chan) return 0; @@ -233,8 +240,22 @@ static int dcpaud_init_dma(struct dcp_audio *dcpaud) } dcpaud->chan = chan; + return 0; +} + +static int dcpaud_init_dma(struct dcp_audio *dcpaud) +{ + int ret; - snd_pcm_set_managed_buffer(dcpaud->substream, SNDRV_DMA_TYPE_DEV_IRAM, - dcpaud->chan->device->dev, 1024 * 1024, - SIZE_MAX); + ret = dcpaud_request_dma(dcpaud); + if (ret) + return ret; + + if (!dcpaud->dma_buffer_configured) { + snd_pcm_set_managed_buffer(dcpaud->substream, + SNDRV_DMA_TYPE_DEV_IRAM, + dcpaud->chan->device->dev, + 1024 * 1024, SIZE_MAX); + dcpaud->dma_buffer_configured = true; + } return 0; @@ -252,6 +273,8 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) mutex_lock(&dcpaud->data_lock); ret = dcpaud_init_dma(dcpaud); - if (ret < 0) + if (ret < 0) { + mutex_unlock(&dcpaud->data_lock); return ret; + } if (!dcpaud->connected) { @@ -278,5 +301,5 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) hw = dcp_pcm_hw; hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED; + SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BATCH; hw.periods_min = 2; hw.periods_max = UINT_MAX; @@ -291,5 +314,32 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) substream->runtime->hw = hw; - return snd_dmaengine_pcm_open(substream, dcpaud->chan); + ret = snd_dmaengine_pcm_open(substream, dcpaud->chan); + if (ret) + return ret; + + /* Native 26.6 keeps the physical audio link fixed at 48 kHz and + * resamples CoreAudio clients into it. We retain that link across ALSA + * clients below, so accepting a different rate here would leave SIO and + * the already-started DCP link running at different rates. */ + if (of_device_is_compatible(dcpaud->dev->of_node, + "apple,t8132-dpaudio")) { + ret = snd_pcm_hw_constraint_single(substream->runtime, + SNDRV_PCM_HW_PARAM_RATE, 48000); + /* A positive return means the interval was successfully narrowed. */ + if (ret < 0) + goto err_close; + + ret = snd_pcm_hw_constraint_step(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_PERIOD_BYTES, + SZ_4K); + if (ret) + goto err_close; + } + + return 0; + +err_close: + snd_dmaengine_pcm_close(substream); + return ret; } @@ -318,4 +368,6 @@ static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, struct dma_slave_config slave_config; struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); + bool acquired_power = false; + bool just_powered = false; int ret; @@ -329,9 +381,35 @@ static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, return -EINVAL; + /* T8132 needs the DPA power domain up while AVService prepares the + * link, not merely when the final PCM trigger starts DMA. Native 26.6 + * keeps that hardware session resident across CoreAudio clients and + * continuously feeds silence, so retain the DPA reference across ALSA + * PCM closes rather than invalidating the AV binding after every sound. */ + if (!dcpaud->power_held) { + ret = pm_runtime_get_if_active(dcpaud->dev); + if (ret < 0) + return ret; + if (!ret) { + /* Native closes the stale AV binding while DPA is inactive, + * then powers DPA up, configures SIO, and opens it again. */ + ret = dcp_audiosrv_close_for_power_transition( + dcpaud->dcp_dev); + if (ret < 0) + return ret; + + ret = pm_runtime_resume_and_get(dcpaud->dev); + if (ret < 0) + return ret; + just_powered = true; + } + acquired_power = true; + dcpaud->power_held = true; + } + memset(&slave_config, 0, sizeof(slave_config)); ret = snd_hwparams_to_dma_slave_config(substream, params, &slave_config); dev_info(dcpaud->dev, "snd_hwparams_to_dma_slave_config: %d\n", ret); if (ret < 0) - return ret; + goto rpm_put; slave_config.direction = DMA_MEM_TO_DEV; @@ -344,4 +422,22 @@ static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, ret = dmaengine_slave_config(chan, &slave_config); dev_info(dcpaud->dev, "dmaengine_slave_config: %d\n", ret); + if (ret < 0) + goto rpm_put; + + /* Native 26.6 configures SIO channel 0x64 before issuing another AV audio + * open after a DPA power transition. */ + if (just_powered) { + ret = dcp_audiosrv_open_after_power_transition(dcpaud->dcp_dev); + if (ret < 0) + goto rpm_put; + } + return ret; + +rpm_put: + if (acquired_power) { + pm_runtime_mark_last_busy(dcpaud->dev); + pm_runtime_put_autosuspend(dcpaud->dev); + dcpaud->power_held = false; + } return ret; } @@ -350,9 +446,17 @@ static int dcp_pcm_hw_free(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; + int ret = 0; - if (!dcpaud_connection_up(dcpaud)) + if (of_device_is_compatible(dcpaud->dev->of_node, + "apple,t8132-dpaudio") && + dcpaud->link_started) return 0; - return dcp_audiosrv_unprepare(dcpaud->dcp_dev); + if (dcpaud_connection_up(dcpaud)) { + ret = dcp_audiosrv_unprepare(dcpaud->dcp_dev); + dev_dbg(dcpaud->dev, "AVService unprepare returned %d\n", ret); + } + + return ret; } @@ -360,10 +464,19 @@ static int dcp_pcm_prepare(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; + int ret; if (!dcpaud_connection_up(dcpaud)) return -ENXIO; - return dcp_audiosrv_prepare(dcpaud->dcp_dev, - &dcpaud->selected_cookie); + /* T8132/26.6 submits SIO DMA before AVService prepare/start. Defer + * prepare to the START trigger so that ordering can be preserved. */ + if (of_device_is_compatible(dcpaud->dev->of_node, + "apple,t8132-dpaudio")) + return 0; + + ret = dcp_audiosrv_prepare(dcpaud->dcp_dev, + &dcpaud->selected_cookie); + dev_dbg(dcpaud->dev, "AVService prepare returned %d\n", ret); + return ret; } @@ -371,15 +484,52 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) { struct dcp_audio *dcpaud = substream->pcm->private_data; + bool t8132 = of_device_is_compatible(dcpaud->dev->of_node, + "apple,t8132-dpaudio"); int ret; switch (cmd) { case SNDRV_PCM_TRIGGER_START: + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + if (t8132) { + /* Native 26.6 has audio DMA pending before it prepares and + * starts the DCP link. Starting the link first leaves T8132 + * acknowledging SIO descriptors without consuming them. */ + ret = snd_dmaengine_pcm_trigger(substream, cmd); + if (ret < 0) + return ret; + if (dcpaud->link_started) + return 0; + + ret = dcp_audiosrv_prepare(dcpaud->dcp_dev, + &dcpaud->selected_cookie); + dev_dbg(dcpaud->dev, + "AVService deferred prepare returned %d\n", ret); + if (ret < 0) + goto stop_dma; + } + + ret = dcp_audiosrv_startlink(dcpaud->dcp_dev, + &dcpaud->selected_cookie); + dev_dbg(dcpaud->dev, "AVService startLink returned %d\n", ret); + if (ret < 0) { + if (t8132) + goto stop_dma; + return ret; + } + if (t8132) { + dcpaud->link_started = true; + return 0; + } + break; + case SNDRV_PCM_TRIGGER_RESUME: if (!dcpaud_connection_up(dcpaud)) return -ENXIO; - WARN_ON(pm_runtime_get_sync(dcpaud->dev) < 0); ret = dcp_audiosrv_startlink(dcpaud->dcp_dev, &dcpaud->selected_cookie); + dev_dbg(dcpaud->dev, "AVService startLink returned %d\n", ret); if (ret < 0) return ret; @@ -405,7 +555,8 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_SUSPEND: + if (t8132 && dcpaud->link_started) + return 0; ret = dcp_audiosrv_stoplink(dcpaud->dcp_dev); - pm_runtime_mark_last_busy(dcpaud->dev); - __pm_runtime_put_autosuspend(dcpaud->dev); + dev_dbg(dcpaud->dev, "AVService stopLink returned %d\n", ret); if (ret < 0) return ret; @@ -414,4 +565,8 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) return 0; + +stop_dma: + snd_dmaengine_pcm_trigger(substream, SNDRV_PCM_TRIGGER_STOP); + return ret; } @@ -423,4 +578,5 @@ struct snd_pcm_ops dcp_playback_ops = { .prepare = dcp_pcm_prepare, .trigger = dcp_pcm_trigger, + .sync_stop = snd_dmaengine_pcm_sync_stop, .pointer = snd_dmaengine_pcm_pointer, }; @@ -490,4 +646,5 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) { struct snd_pcm_substream *substream = dcpaud->substream; + bool sync_stop = false; if (!dcpaud->card || dcpaud->connected == connected) { @@ -497,6 +654,13 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) dcpaud->connected = connected; - if (connected) + if (connected) { dcpaud->connection_cookie++; + /* + * The sound card is persistent across HPD, unlike macOS's audio + * device. Admit an already-open stream into the new connection + * generation so its XRUN recovery can build a fresh DCP session. + */ + dcpaud->open_cookie = dcpaud->connection_cookie; + } mutex_unlock(&dcpaud->data_lock); @@ -505,7 +669,21 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) if (!connected) { snd_pcm_stream_lock(substream); - if (substream->runtime) - snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); + if (substream->runtime) { + snd_pcm_stop(substream, SNDRV_PCM_STATE_XRUN); + sync_stop = true; + } snd_pcm_stream_unlock(substream); + + /* + * STOP only schedules apple-sio's TERMINATE64 work. Native waits + * for its ACK and all final reports before DPA/AV teardown, so make + * that boundary explicit before invalidating the DCP link state. + */ + if (sync_stop) + snd_dmaengine_pcm_sync_stop(substream); + + mutex_lock(&dcpaud->data_lock); + dcpaud->link_started = false; + mutex_unlock(&dcpaud->data_lock); } } @@ -543,6 +721,4 @@ static void dcpaud_expose_debugfs_blob(struct dcp_audio *dcpaud, const char *nam #endif -extern bool hdmi_audio; - static int dcpaud_init_snd_card(struct dcp_audio *dcpaud) { @@ -688,4 +864,15 @@ static void dcpaud_comp_unbind(struct device *dev, struct device *main, snd_card_free_when_closed(dcpaud->card); + if (dcpaud->link_started) { + dcp_audiosrv_stoplink(dcpaud->dcp_dev); + dcp_audiosrv_unprepare(dcpaud->dcp_dev); + dcpaud->link_started = false; + } + + if (dcpaud->power_held) { + pm_runtime_put_sync(dcpaud->dev); + dcpaud->power_held = false; + } + if (dcpaud->dma_link) device_link_del(dcpaud->dma_link); @@ -700,4 +887,5 @@ static int dcpaud_probe(struct platform_device *pdev) { struct dcp_audio *dcpaud; + int ret; dcpaud = devm_kzalloc(&pdev->dev, sizeof(*dcpaud), GFP_KERNEL); @@ -719,4 +907,18 @@ static int dcpaud_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dcpaud); + /* DCP 26.6 constructs its AV audio interface against the live SIO + * firmware state. Unlike older systems it cannot recover if SIO probes + * after AVService, so keep the display component graph deferred until + * the DMA provider is actually ready. */ + if (hdmi_audio && of_device_is_compatible(pdev->dev.of_node, + "apple,t8132-dpaudio")) { + ret = dcpaud_request_dma(dcpaud); + if (ret == -ENXIO) + ret = -EPROBE_DEFER; + if (ret) + return dev_err_probe(&pdev->dev, ret, + "waiting for SIO DMA provider\n"); + } + return component_add(&pdev->dev, &dcpaud_comp_ops); } @@ -774,3 +976,2 @@ void __exit dcp_audio_unregister(void) platform_driver_unregister(&dcpaud_driver); } - diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 0d3c752f62d5f5c7d8badd8611c93e440964dd5f..f001733032cb0aac6fc19f1139eb0b26ee5bf2e6 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -70,4 +70,6 @@ struct audiosrv_data { }; +#define DCP_AV_COMPACT_ELEMENTS_MAXSIZE 3072 + static void av_interface_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) @@ -91,4 +93,23 @@ static void av_interface_teardown(struct apple_epic_service *service) } +static void av_aux_interface_teardown(struct apple_epic_service *service) +{ + service->enabled = false; +} + +static int av_interface_call(struct apple_epic_service *service, u16 group, + u32 idx, + const void *data, size_t data_size, + void *reply, size_t reply_size) +{ + if (group) + return -ENOSYS; + + if (reply && reply_size) + memset(reply, 0, reply_size); + + return 0; +} + static void av_audiosrv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) @@ -117,4 +138,5 @@ static void av_audiosrv_teardown(struct apple_epic_service *service) down_write(&asrv->srv_rwsem); asrv->srv = NULL; + asrv->is_open = false; up_write(&asrv->srv_rwsem); @@ -126,15 +148,82 @@ static void av_audiosrv_teardown(struct apple_epic_service *service) } -int dcp_audiosrv_prepare(struct device *dev, struct dcp_sound_cookie *cookie) +int dcp_audiosrv_close_for_power_transition(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); struct audiosrv_data *asrv = dcp->audiosrv; - int ret; + struct apple_epic_service *service; + int ret = 0; + + if (!asrv || dcp->fw_compat != DCP_FIRMWARE_V_26_6) + return 0; down_write(&asrv->srv_rwsem); - ret = afk_service_call(asrv->srv, 0, asrv->cmds.prepare, cookie, - sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, - 64); + service = asrv->srv; + if (!service) { + ret = -ENODEV; + goto out; + } + if (!asrv->is_open) + goto out; + + /* Native 26.6 closes the AV binding after DPA has gone inactive and + * before powering it back up. Closing a live binding instead tears + * down the complete IOAV display interface. */ + ret = afk_service_call(service, 0, asrv->cmds.close, NULL, 0, 16, + NULL, 0, 16); + if (!ret) + asrv->is_open = false; + +out: up_write(&asrv->srv_rwsem); + return ret; +} + +int dcp_audiosrv_open_after_power_transition(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + struct apple_epic_service *service; + int ret = 0; + + if (!asrv || dcp->fw_compat != DCP_FIRMWARE_V_26_6) + return 0; + + down_write(&asrv->srv_rwsem); + service = asrv->srv; + if (!service) { + ret = -ENODEV; + goto out; + } + + /* This is the matching half of close_for_power_transition(): DPA is up + * and SIO has been configured before native issues open. */ + ret = afk_service_call(service, 0, asrv->cmds.open, NULL, 0, 32, + NULL, 0, 32); + if (!ret) + asrv->is_open = true; + +out: + up_write(&asrv->srv_rwsem); + return ret; +} + +int dcp_audiosrv_prepare(struct device *dev, struct dcp_sound_cookie *cookie) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + if (!asrv) + return -ENODEV; + + down_read(&asrv->srv_rwsem); + if (asrv->srv) + ret = afk_service_call(asrv->srv, 0, asrv->cmds.prepare, cookie, + sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, + 64); + else + ret = -ENODEV; + up_read(&asrv->srv_rwsem); return ret; @@ -147,9 +236,15 @@ int dcp_audiosrv_startlink(struct device *dev, struct dcp_sound_cookie *cookie) int ret; - down_write(&asrv->srv_rwsem); - ret = afk_service_call(asrv->srv, 0, asrv->cmds.start_link, cookie, - sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, - 64); - up_write(&asrv->srv_rwsem); + if (!asrv) + return -ENODEV; + + down_read(&asrv->srv_rwsem); + if (asrv->srv) + ret = afk_service_call(asrv->srv, 0, asrv->cmds.start_link, cookie, + sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, + 64); + else + ret = -ENODEV; + up_read(&asrv->srv_rwsem); return ret; @@ -162,8 +257,14 @@ int dcp_audiosrv_stoplink(struct device *dev) int ret; - down_write(&asrv->srv_rwsem); - ret = afk_service_call(asrv->srv, 0, asrv->cmds.stop_link, NULL, 0, 64, - NULL, 0, 64); - up_write(&asrv->srv_rwsem); + if (!asrv) + return -ENODEV; + + down_read(&asrv->srv_rwsem); + if (asrv->srv) + ret = afk_service_call(asrv->srv, 0, asrv->cmds.stop_link, NULL, 0, + 64, NULL, 0, 64); + else + ret = -ENODEV; + up_read(&asrv->srv_rwsem); return ret; @@ -176,8 +277,14 @@ int dcp_audiosrv_unprepare(struct device *dev) int ret; - down_write(&asrv->srv_rwsem); - ret = afk_service_call(asrv->srv, 0, asrv->cmds.unprepare, NULL, 0, 64, - NULL, 0, 64); - up_write(&asrv->srv_rwsem); + if (!asrv) + return -ENODEV; + + down_read(&asrv->srv_rwsem); + if (asrv->srv) + ret = afk_service_call(asrv->srv, 0, asrv->cmds.unprepare, NULL, 0, + 64, NULL, 0, 64); + else + ret = -ENODEV; + up_read(&asrv->srv_rwsem); return ret; @@ -209,5 +316,5 @@ dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, bfr, sizeof(*hdr) + output_maxsize, 0); if (ret) - return ret; + goto out; if (output) @@ -217,5 +324,7 @@ dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, *output_size = le64_to_cpu(hdr->used_size); - return 0; +out: + kfree(bfr); + return ret; } @@ -224,16 +333,32 @@ int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize struct apple_dcp *dcp = dev_get_drvdata(dev); struct audiosrv_data *asrv = dcp->audiosrv; - size_t size; + size_t request_maxsize = maxsize; + size_t size = 0; int ret; - down_write(&asrv->srv_rwsem); - ret = dcp_audiosrv_osobject_call(asrv->srv, 1, asrv->cmds.get_elements, - elements, maxsize, &size); - up_write(&asrv->srv_rwsem); + if (!asrv) + return -ENODEV; + + memset(elements, 0, maxsize); + down_read(&asrv->srv_rwsem); + if (!asrv->srv) { + ret = -ENODEV; + } else { + /* Native advertises a much larger buffer and fragments the compact + * transaction. The 26.6 response observed on J773g is 1952 bytes, + * so keep this request within one AFK ring entry. */ + if (asrv->srv->ep->compact) + request_maxsize = min_t(size_t, request_maxsize, + DCP_AV_COMPACT_ELEMENTS_MAXSIZE); + ret = dcp_audiosrv_osobject_call(asrv->srv, 1, + asrv->cmds.get_elements, elements, + request_maxsize, &size); + } + up_read(&asrv->srv_rwsem); - if (ret && asrv->warned_get_elements) { + if (ret && !asrv->warned_get_elements) { dev_err(dev, "audiosrv: error getting elements: %d\n", ret); asrv->warned_get_elements = true; - } else { + } else if (!ret) { dev_dbg(dev, "audiosrv: got %zd bytes worth of elements\n", size); } @@ -246,17 +371,24 @@ int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsi struct apple_dcp *dcp = dev_get_drvdata(dev); struct audiosrv_data *asrv = dcp->audiosrv; - size_t size; + size_t size = 0; int ret; - down_write(&asrv->srv_rwsem); - ret = dcp_audiosrv_osobject_call(asrv->srv, 1, - asrv->cmds.get_product_attrs, attrs, - maxsize, &size); - up_write(&asrv->srv_rwsem); + if (!asrv) + return -ENODEV; - if (ret && asrv->warned_get_product_attrs) { + memset(attrs, 0, maxsize); + down_read(&asrv->srv_rwsem); + if (asrv->srv) + ret = dcp_audiosrv_osobject_call(asrv->srv, 1, + asrv->cmds.get_product_attrs, attrs, + maxsize, &size); + else + ret = -ENODEV; + up_read(&asrv->srv_rwsem); + + if (ret && !asrv->warned_get_product_attrs) { dev_err(dev, "audiosrv: error getting product attributes: %d\n", ret); asrv->warned_get_product_attrs = true; - } else { + } else if (!ret) { dev_dbg(dev, "audiosrv: got %zd bytes worth of product attributes\n", size); } @@ -280,4 +412,5 @@ static const struct apple_epic_service_ops avep_ops[] = { .name = "DCPAVSimpleVideoInterface", .init = av_interface_init, + .call = av_interface_call, .teardown = av_interface_teardown, }, @@ -285,7 +418,27 @@ static const struct apple_epic_service_ops avep_ops[] = { .name = "DCPAVAudioInterface", .init = av_audiosrv_init, + .call = av_interface_call, + .report = av_audiosrv_report, + .teardown = av_audiosrv_teardown, + }, + { + .name = "dcpav-video-interface-", + .init = av_interface_init, + .call = av_interface_call, + .teardown = av_aux_interface_teardown, + }, + { + .name = "dcpav-audio-interface-", + .init = av_audiosrv_init, + .call = av_interface_call, .report = av_audiosrv_report, .teardown = av_audiosrv_teardown, }, + { + .name = "dcpav-cec-interface-ep", + .init = av_interface_init, + .call = av_interface_call, + .teardown = av_aux_interface_teardown, + }, {} }; @@ -394,4 +547,5 @@ int avep_init(struct apple_dcp *dcp) break; case DCP_FIRMWARE_V_13_5: + case DCP_FIRMWARE_V_26_6: audiosrv_data->cmds = dcp_av_audio_cmds_v13_5; break; @@ -412,5 +566,5 @@ int avep_init(struct apple_dcp *dcp) of_node_put(audio_node); dev_info(dev, "No audio support\n"); - return 0; + goto start_ep; } @@ -419,5 +573,5 @@ int avep_init(struct apple_dcp *dcp) if (!audio_pdev) { dev_info(dev, "No DP/HDMI audio device not ready\n"); - return 0; + goto start_ep; } dcp->audiosrv->audio_dev = audio_pdev; @@ -426,4 +580,5 @@ int avep_init(struct apple_dcp *dcp) DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); +start_ep: dcp->avep = afk_init(dcp, AV_ENDPOINT, avep_ops); if (IS_ERR(dcp->avep)) diff --git a/drivers/gpu/drm/apple/av.h b/drivers/gpu/drm/apple/av.h index c00cbef549fd2e637132eb9df3c7d1910babec03..ac0556a8ff8d8d928b152a675ce13914ab2a2289 100644 --- a/drivers/gpu/drm/apple/av.h +++ b/drivers/gpu/drm/apple/av.h @@ -10,7 +10,17 @@ void av_service_connect(struct apple_dcp *dcp); void av_service_disconnect(struct apple_dcp *dcp); +int dcp_audiosrv_close_for_power_transition(struct device *dev); +int dcp_audiosrv_open_after_power_transition(struct device *dev); #else static inline void av_service_connect(struct apple_dcp *dcp) { } static inline void av_service_disconnect(struct apple_dcp *dcp) { } +static inline int dcp_audiosrv_close_for_power_transition(struct device *dev) +{ + return 0; +} +static inline int dcp_audiosrv_open_after_power_transition(struct device *dev) +{ + return 0; +} #endif diff --git a/drivers/gpu/drm/apple/connector.c b/drivers/gpu/drm/apple/connector.c index 15b3664d85631e5e6eb8f74fc8b3818dcdfd8657..a6d2441b16222595d6c8e75c6fabe27cd436c681 100644 --- a/drivers/gpu/drm/apple/connector.c +++ b/drivers/gpu/drm/apple/connector.c @@ -5,4 +5,5 @@ #include "connector.h" +#include "dcp.h" #include "linux/err.h" @@ -80,4 +81,147 @@ CONNECTOR_DEBUGFS_ENTRY(display_attribs, DCP_CHUNK_DISPLAY_ATTRIBUTES); CONNECTOR_DEBUGFS_ENTRY(transport, DCP_CHUNK_TRANSPORT); +static ssize_t dptx_connect_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct apple_connector *apple_con = file->private_data; + int ret; + + ret = dcp_dptx_connect_oob(apple_con->dcp, 0); + return ret ? ret : len; +} + +static const struct file_operations dptx_connect_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dptx_connect_write, + .llseek = noop_llseek, +}; + +static ssize_t dptx_disconnect_write(struct file *file, + const char __user *buf, size_t len, + loff_t *ppos) +{ + struct apple_connector *apple_con = file->private_data; + int ret; + + ret = dcp_dptx_disconnect_oob(apple_con->dcp, 0); + return ret ? ret : len; +} + +static const struct file_operations dptx_disconnect_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dptx_disconnect_write, + .llseek = noop_llseek, +}; + +static ssize_t dptx_mark_disconnected_write(struct file *file, + const char __user *buf, size_t len, + loff_t *ppos) +{ + struct apple_connector *apple_con = file->private_data; + + dcp_hotplug_mark_disconnected_oob(apple_con->dcp); + return len; +} + +static const struct file_operations dptx_mark_disconnected_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dptx_mark_disconnected_write, + .llseek = noop_llseek, +}; + +static ssize_t dptx_av_disconnect_write(struct file *file, + const char __user *buf, size_t len, + loff_t *ppos) +{ + struct apple_connector *apple_con = file->private_data; + + dcp_av_disconnect_oob(apple_con->dcp); + return len; +} + +static const struct file_operations dptx_av_disconnect_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dptx_av_disconnect_write, + .llseek = noop_llseek, +}; + +static ssize_t dptx_hpd_low_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct apple_connector *apple_con = file->private_data; + int ret; + + ret = dcp_dptx_set_hpd_oob(apple_con->dcp, 0, false); + return ret ? ret : len; +} + +static const struct file_operations dptx_hpd_low_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dptx_hpd_low_write, + .llseek = noop_llseek, +}; + +static ssize_t dptx_hpd_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct apple_connector *apple_con = file->private_data; + bool hpd; + int ret; + + ret = kstrtobool_from_user(buf, len, &hpd); + if (ret) + return ret; + + ret = dcp_dptx_set_hpd_oob(apple_con->dcp, 0, hpd); + return ret ? ret : len; +} + +static const struct file_operations dptx_hpd_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dptx_hpd_write, + .llseek = noop_llseek, +}; + +static ssize_t dptx_release_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct apple_connector *apple_con = file->private_data; + int ret; + + ret = dcp_dptx_release_oob(apple_con->dcp, 0); + return ret ? ret : len; +} + +static const struct file_operations dptx_release_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dptx_release_write, + .llseek = noop_llseek, +}; + +static ssize_t dptx_phy_activate_write(struct file *file, + const char __user *buf, size_t len, + loff_t *ppos) +{ + struct apple_connector *apple_con = file->private_data; + int ret; + + ret = dcp_dptx_phy_activate_oob(apple_con->dcp); + return ret ? ret : len; +} + +static const struct file_operations dptx_phy_activate_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = dptx_phy_activate_write, + .llseek = noop_llseek, +}; + static void dcp_afk_debugfs_root(struct platform_device *pdev, int ep, struct dentry *root) { @@ -116,4 +260,20 @@ void apple_connector_debugfs_init(struct drm_connector *connector, struct dentry case DRM_MODE_CONNECTOR_HDMIA: dcp_afk_debugfs_root(apple_con->dcp, AV_ENDPOINT, root); + debugfs_create_file("dptx_connect", 0200, root, apple_con, + &dptx_connect_fops); + debugfs_create_file("dptx_disconnect", 0200, root, apple_con, + &dptx_disconnect_fops); + debugfs_create_file("dptx_mark_disconnected", 0200, root, + apple_con, &dptx_mark_disconnected_fops); + debugfs_create_file("dptx_av_disconnect", 0200, root, apple_con, + &dptx_av_disconnect_fops); + debugfs_create_file("dptx_hpd_low", 0200, root, apple_con, + &dptx_hpd_low_fops); + debugfs_create_file("dptx_hpd", 0200, root, apple_con, + &dptx_hpd_fops); + debugfs_create_file("dptx_release", 0200, root, apple_con, + &dptx_release_fops); + debugfs_create_file("dptx_phy_activate", 0200, root, apple_con, + &dptx_phy_activate_fops); break; default: diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 57832487d9a4ab72233dbe0ccebfad498c8a114c..53827fb81cb31947c96437445c884daad255abbe 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -19,4 +19,5 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" +#include "iomfb_v26_6.h" #include "epic/dpavservep.h" @@ -26,4 +27,6 @@ struct apple_dcp; struct apple_dcp_afkep; +extern bool dcp_disable_rt_bandwidth; + struct dcpav_service_epic; @@ -32,4 +35,5 @@ enum dcp_firmware_version { DCP_FIRMWARE_V_12_3, DCP_FIRMWARE_V_13_5, + DCP_FIRMWARE_V_26_6, }; @@ -39,8 +43,13 @@ enum { DCP_EXPERT_ENDPOINT = 0x22, DISP0_ENDPOINT = 0x23, + DPAVCTRL_ENDPOINT = 0x24, + EPIC25_ENDPOINT = 0x25, + DPSAC_ENDPOINT = 0x26, + DPDEV_ENDPOINT = 0x27, DPAVSERV_ENDPOINT = 0x28, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, HDCP_ENDPOINT = 0x2b, + EPIC2C_ENDPOINT = 0x2c, REMOTE_ALLOC_ENDPOINT = 0x2d, IOMFB_ENDPOINT = 0x37, @@ -179,4 +188,5 @@ struct apple_dcp { struct dcp_swap_submit_req_v12_3 v12_3; struct dcp_swap_submit_req_v13_3 v13_3; + struct dcp_swap_submit_req_v26_6_0 v26_6_0; } swap; @@ -188,4 +198,11 @@ struct apple_dcp { bool during_modeset; bool valid_mode; + /* + * Diagnostic 26.6 external-hotplug path: the link can disappear while + * IOMFB remains powered. Remember that atomic_disable deliberately + * skipped the firmware power-off sequence so atomic_enable does not send + * a mismatched power-on sequence on reconnect. + */ + bool hotplug_keepalive; bool use_timestamps; struct dcp_set_digital_out_mode_req mode; @@ -240,4 +257,23 @@ struct apple_dcp { struct apple_dcp_afkep *ibootep; + struct apple_epic_service *iboot_service; + struct apple_dcp_afkep *dpavctrlep; + struct apple_dcp_afkep *dpdevep; + struct apple_epic_service *dpdev_service; + struct completion dpdev_ready; + struct apple_dcp_afkep *avauxep; + struct apple_dcp_afkep *dpsacep; + struct apple_dcp_afkep *remoteallocep; + struct apple_dcp_afkep *epicep2c; + struct apple_dcp_afkep *expertep; + struct apple_dcp_afkep *epicep25; + struct apple_dcp_afkep *hdcpep; + struct completion hdcp_ready; + struct apple_epic_service *hdcp_service[2]; + u32 hdcp_teardown_interface; + bool hdcp_mprime_attempted; + struct completion dpavctrl_ready; + struct apple_epic_service *dpavctrl_av_controller[2]; + struct apple_epic_service *dpavctrl_late_service; struct apple_dcp_afkep *dcpavservep; struct dcpavserv dcpavserv; @@ -269,4 +305,6 @@ struct apple_dcp { }; +void *dcp_vmap_wc(phys_addr_t phys, size_t size, void **map_base); + void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 272d1cef38e66134d899e1ec5e50db1e047cf628..d427094d914c1f321ba80580774088fd397427b6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -8,4 +8,5 @@ #include #include +#include #include #include @@ -25,11 +26,14 @@ #include #include +#include #include #include +#include #include #include #include #include +#include #include @@ -55,4 +59,19 @@ module_param(hdmi_audio, bool, 0644); MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); +static bool enable_hdcp; +module_param(enable_hdcp, bool, 0644); +MODULE_PARM_DESC(enable_hdcp, + "Enable experimental 26.6 HDCP endpoint support"); + +static bool enable_dpavctrl; +module_param(enable_dpavctrl, bool, 0644); +MODULE_PARM_DESC(enable_dpavctrl, + "Activate experimental 26.6 DPAV link-control services"); + +bool dcp_disable_rt_bandwidth; +module_param_named(disable_rt_bandwidth, dcp_disable_rt_bandwidth, bool, 0644); +MODULE_PARM_DESC(disable_rt_bandwidth, + "Return no Dashboard registers from the D003 RT-bandwidth callback"); + static bool unstable_edid = true; module_param(unstable_edid, bool, 0644); @@ -230,5 +249,5 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) return iomfb_recv_msg(dcp, message); case AV_ENDPOINT: - afk_receive_message(dcp->avep, message); + afk_receive_message(dcp->avauxep ?: dcp->avep, message); return; case SYSTEM_ENDPOINT: @@ -238,4 +257,29 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) afk_receive_message(dcp->ibootep, message); return; + case DPAVCTRL_ENDPOINT: + afk_receive_message(dcp->dpavctrlep, message); + return; + case DPDEV_ENDPOINT: + afk_receive_message(dcp->dpdevep, message); + return; + case DPSAC_ENDPOINT: + afk_receive_message(dcp->dpsacep, message); + return; + case REMOTE_ALLOC_ENDPOINT: + afk_receive_message(dcp->remoteallocep, message); + return; + case EPIC2C_ENDPOINT: + afk_receive_message(dcp->epicep2c, message); + return; + case DCP_EXPERT_ENDPOINT: + afk_receive_message(dcp->expertep, message); + return; + case EPIC25_ENDPOINT: + afk_receive_message(dcp->epicep25, message); + return; + case HDCP_ENDPOINT: + if (dcp->hdcpep) + afk_receive_message(dcp->hdcpep, message); + return; case DPAVSERV_ENDPOINT: afk_receive_message(dcp->dcpavservep, message); @@ -254,4 +298,6 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dcp->crashed = true; + if (dcp->dpdevep) + afk_cancel_commands(dcp->dpdevep); dev_err(dcp->dev, "DCP has crashed\n"); if (dcp->connector) { @@ -264,4 +310,28 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ } +void *dcp_vmap_wc(phys_addr_t phys, size_t size, void **map_base) +{ + unsigned long offset = offset_in_page(phys); + unsigned int count = DIV_ROUND_UP(offset + size, PAGE_SIZE); + unsigned long first_pfn = PHYS_PFN(phys); + struct page **pages; + void *base; + unsigned int i; + + pages = kcalloc(count, sizeof(*pages), GFP_KERNEL); + if (!pages) + return NULL; + for (i = 0; i < count; i++) + pages[i] = pfn_to_page(first_pfn + i); + + base = vmap(pages, count, VM_MAP, pgprot_writecombine(PAGE_KERNEL)); + kfree(pages); + if (!base) + return NULL; + + *map_base = base; + return base + offset; +} + static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) { @@ -281,7 +351,10 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) return -ENOMEM; - // TODO: verify phy_addr, cache attribute - bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); - if (!bfr->buffer) + /* Retained DCP buffers may be accessed through a realtime path. + * A WB alias creates AMCC directory state that can fault when DCP + * subsequently accesses the same memory. */ + bfr->iomem = (void __iomem *)dcp_vmap_wc(phy_addr, bfr->size, + &bfr->private); + if (!bfr->iomem) return -ENOMEM; @@ -290,5 +363,5 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx\n", (uintptr_t)bfr->iova, (uintptr_t)phy_addr, - (uintptr_t)bfr->buffer); + (uintptr_t)bfr->iomem); } else { bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, @@ -309,5 +382,5 @@ static void dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) if (bfr->is_mapped) - memunmap(bfr->buffer); + vunmap(bfr->private); else dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); @@ -333,10 +406,31 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) struct apple_dcp *dcp = platform_get_drvdata(pdev); struct drm_crtc_state *crtc_state; + struct drm_plane *plane; + const struct drm_plane_state *plane_state; + u32 max_layers; + u32 active_layers = 0; bool needs_modeset; - if (dcp->crashed) + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + /* A failed auxiliary DCP must not poison atomic transactions for the + * other display pipelines. Reject attempts to drive the crashed DCP, + * but permit its already-disabled CRTC to remain disabled. */ + if (dcp->crashed && crtc_state->active) return -EINVAL; - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + if (!of_property_read_u32(pdev->dev.of_node, "apple,iomfb-max-layers", + &max_layers)) { + drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, + crtc_state) { + if (plane_state->fb && plane_state->visible) + active_layers++; + } + if (active_layers > max_layers) { + drm_dbg_kms(crtc->dev, + "rejecting %u active DCP layers (maximum %u)\n", + active_layers, max_layers); + return -EINVAL; + } + } needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; @@ -380,6 +474,20 @@ static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) reinit_completion(&dcp->dptxport[port].linkcfg_completion); dcp->dptxport[port].atcphy = dcp->phy; - dptxport_connect(dcp->dptxport[port].service, 0, dcp->dptx_phy, dcp->dptx_die); - dptxport_request_display(dcp->dptxport[port].service); + dev_info(dcp->dev, "DPTX port %d: connecting remote port\n", port); + ret = dptxport_connect(dcp->dptxport[port].service, 0, + dcp->dptx_phy, dcp->dptx_die); + if (ret) { + dev_warn(dcp->dev, "DPTX port %d: connect failed: %d\n", + port, ret); + goto out_unlock; + } + + dev_info(dcp->dev, "DPTX port %d: requesting display\n", port); + ret = dptxport_request_display(dcp->dptxport[port].service); + if (ret) { + dev_warn(dcp->dev, "DPTX port %d: display request failed: %d\n", + port, ret); + goto out_unlock; + } dcp->dptxport[port].connected = true; @@ -437,4 +545,14 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) } +int dcp_dptx_phy_activate_oob(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (!dcp->phy) + return -ENODEV; + + return phy_set_mode_ext(dcp->phy, PHY_MODE_DP, dcp->index); +} + int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) { @@ -452,4 +570,40 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) } +void dcp_hotplug_mark_disconnected_oob(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + disconnected_hpd_event(dcp->connector); +} + +void dcp_av_disconnect_oob(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp->avep) + av_service_disconnect(dcp); +} + +int dcp_dptx_set_hpd_oob(struct platform_device *pdev, u32 port, bool hpd) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (port >= dcp->hw.num_dptx_ports || !dcp->dptxport[port].enabled) + return -ENODEV; + + dev_info(dcp->dev, "%s(port=%u, hpd=%d)\n", __func__, port, hpd); + return dptxport_set_hpd(dcp->dptxport[port].service, hpd); +} + +int dcp_dptx_release_oob(struct platform_device *pdev, u32 port) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (port >= dcp->hw.num_dptx_ports) + return -EINVAL; + + return dcp_dptx_disconnect(dcp, port); +} + static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) { @@ -501,11 +655,344 @@ unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) } +static void dcp_compact_aux_init(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + + if (service->ep->endpoint == DPDEV_ENDPOINT && name && + !strcmp(name, "dcpdp-device-epic")) { + if (dcp->dpdev_service && dcp->dpdev_service != service) + dcp->dpdev_service->enabled = false; + dcp->dpdev_service = service; + complete_all(&dcp->dpdev_ready); + } + + dev_dbg(service->ep->dcp->dev, + "AFK[ep:%02x]: compact auxiliary service %s (%s, unit %lld)\n", + service->ep->endpoint, name ?: "", class ?: "", unit); +} + +static int dcp_compact_aux_call(struct apple_epic_service *service, u16 group, + u32 idx, + const void *data, size_t data_size, + void *reply, size_t reply_size) +{ + if (group) + return -ENOSYS; + + if (reply && reply_size) + memset(reply, 0, reply_size); + return 0; +} + +static int dcp_hdcp_read_mprime(struct apple_dcp *dcp) +{ + __le64 request[12] = { 0 }; + u8 response[sizeof(request)] = { 0 }; + int ret; + + if (!dcp->dpdev_service) + return -ENODEV; + + request[0] = cpu_to_le64(0x69473); + request[2] = cpu_to_le64(0x20); + request[4] = cpu_to_le64(0x1f4); + + ret = afk_service_call(dcp->dpdev_service, 1, 6, + request, sizeof(request), 0, + response, sizeof(response), 0); + if (ret) { + dev_err(dcp->dev, "HDCP teardown MPRIME read failed: %d\n", ret); + return ret; + } + + /* + * This call is on the time-critical unplug path. In particular, do not + * dump the response here: synchronous console printing can delay the + * following HDCP notification long enough for the link teardown to fail. + */ + dev_dbg(dcp->dev, "HDCP teardown MPRIME read completed\n"); + return 0; +} + +static int dcp_hdcp_notify_dpdev(struct apple_dcp *dcp) +{ + __le64 request[2] = { 0 }; + __le64 response[2] = { 0 }; + int ret; + + if (!dcp->dpdev_service) + return -ENODEV; + + /* + * Native 26.6 performs this DPDEV transaction synchronously while + * handling command 8 for each half of an HDCP auth-session pair. In + * particular it happens before the command 8 reply on unplug. Skipping + * it lets the services tear down, but DCP subsequently asserts while + * finishing link shutdown. + */ + ret = afk_service_call(dcp->dpdev_service, 0, 5, + request, sizeof(request), 0, + response, sizeof(response), 0); + if (ret) + dev_err(dcp->dev, "HDCP DPDEV notification failed: %d\n", ret); + + return ret; +} + +static int dcp_hdcp_call(struct apple_epic_service *service, u16 group, u32 idx, + const void *data, size_t data_size, + void *reply, size_t reply_size) +{ + struct apple_dcp *dcp = service->ep->dcp; + __le64 ready = cpu_to_le64(1); + + if (group) + return -ENOSYS; + + if (reply && reply_size) { + memset(reply, 0, reply_size); + if (data && data_size) + memcpy(reply, data, min(data_size, reply_size)); + } + + /* + * Command 7 asks whether the upper half of an auth-session pair is + * ready. Native 26.6 returns one in the third u64; returning an all-zero + * blob leaves firmware polling this callback forever and prevents it + * from publishing the active 9/11 session pair. + */ + if (idx == 7 && (service->channel & 3) == 3 && + reply && reply_size >= 3 * sizeof(ready)) + memcpy(reply + 2 * sizeof(ready), &ready, sizeof(ready)); + + /* Keep the nested DPDEV notification ahead of the command 8 reply. */ + if (idx == 8) + dcp_hdcp_notify_dpdev(dcp); + + /* + * Native 26.6 reads the HDCP2 MPRIME register through DPDEV while + * handling command 11 on the upper interface of the active auth-session + * pair. Keep the nested DPDEV call ahead of the command 11 reply. + */ + if (idx == 11 && service->channel == dcp->hdcp_teardown_interface && + !dcp->hdcp_mprime_attempted) { + dcp->hdcp_mprime_attempted = true; + dcp_hdcp_read_mprime(dcp); + } + + return 0; +} + +static const struct apple_epic_service_ops dcp_compact_aux_ops[] = { + { + .name = "*", + .init = dcp_compact_aux_init, + .call = dcp_compact_aux_call, + }, + {}, +}; + +static void dcp_hdcp_service_init(struct apple_epic_service *service, + const char *name, const char *class, + s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + + /* + * Interfaces 1 and 3 are the two HDCP controllers. Native 26.6 + * activates both, on opposite sides of IOMFB startup. + */ + if (service->channel == 1 || service->channel == 3) { + dcp->hdcp_service[service->channel >> 1] = service; + if (dcp->hdcp_service[0] && dcp->hdcp_service[1]) + complete_all(&dcp->hdcp_ready); + } + + if (name && strstr(name, "hdcp-auth-sess") && + (service->channel & 3) == 3) { + dcp->hdcp_teardown_interface = service->channel; + dcp->hdcp_mprime_attempted = false; + } +} + +static const struct apple_epic_service_ops dcp_hdcp_ops[] = { + { + .name = "*", + .init = dcp_hdcp_service_init, + .call = dcp_hdcp_call, + }, + {}, +}; + +static void dcp_dpavctrl_service_init(struct apple_epic_service *service, + const char *name, const char *class, + s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + const char *service_class = class ?: name; + + if (service_class && strstr(service_class, "dcpav-controller-epic")) + dcp->dpavctrl_av_controller[!!(service->channel & BIT(2))] = + service; + + if ((service->channel & 7) == 7) { + service->defer_start = true; + dcp->dpavctrl_late_service = service; + } + + if (service->ep->num_channels >= 4) + complete_all(&dcp->dpavctrl_ready); +} + +static const struct apple_epic_service_ops dcp_dpavctrl_ops[] = { + { + .name = "AppleDCPDPTXController", + .init = dcp_dpavctrl_service_init, + .call = dcp_compact_aux_call, + }, + { + .name = "dcpav-controller-epic", + .init = dcp_dpavctrl_service_init, + .call = dcp_compact_aux_call, + }, + { + .name = "dcpdp-controller-epic", + .init = dcp_dpavctrl_service_init, + .call = dcp_compact_aux_call, + }, + {}, +}; + +static int dcp_start_aux_ep(struct apple_dcp *dcp, u32 endpoint, + struct apple_dcp_afkep **out) +{ + int ret; + + *out = afk_init(dcp, endpoint, dcp_compact_aux_ops); + if (IS_ERR(*out)) { + ret = PTR_ERR(*out); + *out = NULL; + return ret; + } + + ret = afk_start(*out); + if (ret) { + afk_shutdown(*out); + *out = NULL; + } + + return ret; +} + +static int dcp_start_hdcp(struct apple_dcp *dcp) +{ + unsigned long timeout; + int ret; + + init_completion(&dcp->hdcp_ready); + dcp->hdcp_service[0] = NULL; + dcp->hdcp_service[1] = NULL; + dcp->hdcp_teardown_interface = 0; + dcp->hdcp_mprime_attempted = false; + dcp->hdcpep = afk_init(dcp, HDCP_ENDPOINT, dcp_hdcp_ops); + if (IS_ERR(dcp->hdcpep)) { + ret = PTR_ERR(dcp->hdcpep); + dcp->hdcpep = NULL; + return ret; + } + + ret = afk_start(dcp->hdcpep); + if (ret) + return ret; + + timeout = wait_for_completion_timeout(&dcp->hdcp_ready, + msecs_to_jiffies(1000)); + if (!timeout) + return -ETIMEDOUT; + + /* Ensure afk_recv_handle_compact_init() has started the service. */ + flush_workqueue(dcp->hdcpep->wq); + return 0; +} + +static int dcp_activate_hdcp(struct apple_dcp *dcp, unsigned int controller) +{ + __le64 request[4] = { 0, 0, cpu_to_le64(3), 0 }; + + if (controller >= ARRAY_SIZE(dcp->hdcp_service) || + !dcp->hdcp_service[controller]) + return -ENODEV; + + return afk_service_call(dcp->hdcp_service[controller], 4, 6, + request, sizeof(request), 0, + NULL, 0, sizeof(request)); +} + +static int dcp_start_dpavctrl(struct apple_dcp *dcp) +{ + unsigned long timeout; + int ret; + + init_completion(&dcp->dpavctrl_ready); + dcp->dpavctrlep = afk_init(dcp, DPAVCTRL_ENDPOINT, + dcp_dpavctrl_ops); + if (IS_ERR(dcp->dpavctrlep)) { + ret = PTR_ERR(dcp->dpavctrlep); + dcp->dpavctrlep = NULL; + return ret; + } + + ret = afk_start(dcp->dpavctrlep); + if (ret) + return ret; + + timeout = wait_for_completion_timeout(&dcp->dpavctrl_ready, + msecs_to_jiffies(1000)); + if (!timeout) + return -ETIMEDOUT; + + return 0; +} + +static int dcp_activate_dpavctrl(struct apple_dcp *dcp) +{ + int i, ret; + + /* The two AV controllers must be opened before the deferred interface. */ + for (i = 1; i >= 0; i--) { + if (!dcp->dpavctrl_av_controller[i]) + return -ENODEV; + ret = afk_service_call(dcp->dpavctrl_av_controller[i], 0, 12, + NULL, 0, 32, NULL, 0, 32); + if (ret) + return ret; + } + + if (!dcp->dpavctrl_late_service) + return -ENODEV; + + return afk_start_service(dcp->dpavctrl_late_service); +} + int dcp_start(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + bool v26_6 = dcp->fw_compat == DCP_FIRMWARE_V_26_6; int ret; init_completion(&dcp->start_done); + /* + * Do not wake RTKit from component bind. All DCP components are bound + * before appledrm starts their application endpoints, and newer firmware + * does not tolerate being left running with those endpoints unserviced. + * Wake each coprocessor only once its endpoint handlers are ready to run. + */ + ret = apple_rtkit_wake(dcp->rtk); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to acquire RTKit: %d\n", ret); + /* start RTKit endpoints */ ret = systemep_init(dcp); @@ -513,5 +1000,53 @@ int dcp_start(struct platform_device *pdev) dev_warn(dcp->dev, "Failed to start system endpoint: %d\n", ret); - if (unstable_edid && !dcp_has_panel(dcp)) { + /* 26.6 starts the IOMFB transport before constructing its HDMI controller + * graph, but does not bind shared memory until the graph is complete. */ + if (v26_6) { + ret = apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start IOMFB transport endpoint\n"); + + init_completion(&dcp->dpdev_ready); + dcp->dpdev_service = NULL; + ret = dcp_start_aux_ep(dcp, DPDEV_ENDPOINT, &dcp->dpdevep); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start DPDEV endpoint\n"); + + ret = dcp_start_dpavctrl(dcp); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to initialize DPAVCTRL endpoint\n"); + + ret = dcp_start_aux_ep(dcp, DPSAC_ENDPOINT, &dcp->dpsacep); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start DPSAC endpoint\n"); + + /* + * The 26.6 AV endpoint is compact, but its audio service still uses + * the established DCPAV command ABI. Let the audio-aware endpoint + * own 0x29 when requested; starting the generic endpoint here would + * make the later avep_init() call a permanent no-op. + */ +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + if (hdmi_audio) + ret = avep_init(dcp); + else +#endif + ret = dcp_start_aux_ep(dcp, AV_ENDPOINT, &dcp->avauxep); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start AV endpoint\n"); + + ret = dcp_start_aux_ep(dcp, REMOTE_ALLOC_ENDPOINT, + &dcp->remoteallocep); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start REMOTEALLOC endpoint\n"); + } + + if (!v26_6 && unstable_edid && !dcp_has_panel(dcp)) { ret = dpavservep_init(dcp); if (ret) @@ -526,4 +1061,31 @@ int dcp_start(struct platform_device *pdev) ret); + if (v26_6) { + ret = dcp_start_aux_ep(dcp, EPIC2C_ENDPOINT, + &dcp->epicep2c); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start EPIC2C endpoint\n"); + + if (unstable_edid && !dcp_has_panel(dcp)) { + ret = dpavservep_init(dcp); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start DPAVSERV endpoint\n"); + } + + ret = dcp_start_aux_ep(dcp, DCP_EXPERT_ENDPOINT, + &dcp->expertep); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start DCP expert endpoint\n"); + + ret = dcp_start_aux_ep(dcp, EPIC25_ENDPOINT, + &dcp->epicep25); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start EPIC25 endpoint\n"); + } + ret = dptxep_init(dcp); if (ret) { @@ -554,13 +1116,50 @@ int dcp_start(struct platform_device *pdev) #endif } + if (v26_6 && !ret && dcp->dptxport[0].enabled) { + ret = dcp_dptx_connect(dcp, 0); + if (ret) + dev_warn(dcp->dev, + "Failed to connect DPTX port during startup: %d\n", + ret); + } + if (v26_6 && enable_hdcp) { + ret = dcp_start_hdcp(dcp); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start HDCP endpoint\n"); + + ret = dcp_activate_hdcp(dcp, 0); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to activate first HDCP controller\n"); + } + if (v26_6 && enable_dpavctrl) { + /* + * DPAV owns link-control services used during ordinary plug and + * unplug, even when content protection is disabled. Keeping its + * activation under enable_hdcp leaves the deferred controller + * unopened and DCP asserts while stopping the upstream link. + */ + ret = dcp_activate_dpavctrl(dcp); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to activate DPAV controllers\n"); + } } else if (dcp->phy) { dev_warn(dcp->dev, "OS firmware incompatible with dptxport EP\n"); } + ret = iomfb_start_rtkit(dcp); if (ret) dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); + else if (v26_6 && enable_hdcp && dcp->hdcpep) { + ret = dcp_activate_hdcp(dcp, 1); + if (ret) + dev_err(dcp->dev, "Failed to activate second HDCP controller: %d\n", + ret); + } #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) - if (hdmi_audio) { + if (hdmi_audio && !dcp->avep && !dcp->avauxep) { ret = avep_init(dcp); if (ret) @@ -582,4 +1181,7 @@ static void _dcp_poweroff(struct apple_dcp *dcp) iomfb_poweroff_v13_3(dcp); break; + case DCP_FIRMWARE_V_26_6: + iomfb_poweroff_v26_6_0(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -641,4 +1243,7 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) iomfb_sleep_v13_3(dcp); break; + case DCP_FIRMWARE_V_26_6: + iomfb_sleep_v26_6_0(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -651,4 +1256,12 @@ void dcp_poweron(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp->hotplug_keepalive) { + dev_info(dcp->dev, + "reusing live IOMFB state after external hotplug\n"); + dcp->hotplug_keepalive = false; + dcp->valid_mode = false; + return; + } + if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); @@ -666,4 +1279,7 @@ void dcp_poweron(struct platform_device *pdev) iomfb_poweron_v13_3(dcp); break; + case DCP_FIRMWARE_V_26_6: + iomfb_poweron_v26_6_0(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -679,4 +1295,20 @@ void dcp_poweroff(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); + /* + * Keep the 26.6 external IOMFB instance alive while isolating the HPD + * crash. The DPTX/AV link has already been released by the out-of-band + * disconnect path, but the firmware's display-power-off sequence crashes + * after disabling ALPM on T8132. A later atomic enable must modeset the + * still-live instance instead of sending a second power-on sequence. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6 && !dcp->main_display && + dcp->connector && !dcp->connector->connected) { + dev_info(dcp->dev, + "keeping IOMFB alive across external connector disconnect\n"); + dcp->hotplug_keepalive = true; + dcp->valid_mode = false; + return; + } + if (dcp->avep) av_service_disconnect(dcp); @@ -965,4 +1597,6 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) else if (strncmp(compat_str, "13.5.0", sizeof(compat_str)) == 0) return DCP_FIRMWARE_V_13_5; + else if (strncmp(compat_str, "26.6.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_26_6; dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", @@ -1065,5 +1699,4 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); - dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); @@ -1073,4 +1706,5 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); @@ -1078,10 +1712,8 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) return dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); + /* The crash callback can run before dcp_start() initializes IOMFB. */ + init_completion(&dcp->start_done); - ret = apple_rtkit_wake(dcp->rtk); - if (ret) - return dev_err_probe(dev, ret, - "Failed to boot RTKit: %d\n", ret); - return ret; + return 0; } @@ -1100,4 +1732,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) disable_irq(dcp->hdmi_hpd_irq); + if (dcp->dpdevep) + afk_cancel_commands(dcp->dpdevep); + typec_mux_put(dcp->typec_mux); @@ -1108,4 +1743,50 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) } + if (dcp->avauxep) { + afk_shutdown(dcp->avauxep); + dcp->avauxep = NULL; + } + + if (dcp->dpavctrlep) { + afk_shutdown(dcp->dpavctrlep); + dcp->dpavctrlep = NULL; + } + + if (dcp->dpdevep) { + afk_shutdown(dcp->dpdevep); + dcp->dpdevep = NULL; + dcp->dpdev_service = NULL; + } + + if (dcp->dpsacep) { + afk_shutdown(dcp->dpsacep); + dcp->dpsacep = NULL; + } + + if (dcp->remoteallocep) { + afk_shutdown(dcp->remoteallocep); + dcp->remoteallocep = NULL; + } + + if (dcp->epicep2c) { + afk_shutdown(dcp->epicep2c); + dcp->epicep2c = NULL; + } + + if (dcp->expertep) { + afk_shutdown(dcp->expertep); + dcp->expertep = NULL; + } + + if (dcp->epicep25) { + afk_shutdown(dcp->epicep25); + dcp->epicep25 = NULL; + } + + if (dcp->hdcpep) { + afk_shutdown(dcp->hdcpep); + dcp->hdcpep = NULL; + } + if (dcp->dptxep) { afk_shutdown(dcp->dptxep); @@ -1152,4 +1833,12 @@ static const struct component_ops dcp_comp_ops = { }; +static void dcp_phy_power_off(void *data) +{ + struct phy *phy = data; + + phy_power_off(phy); + phy_exit(phy); +} + static int dcp_platform_probe(struct platform_device *pdev) { @@ -1188,4 +1877,24 @@ static int dcp_platform_probe(struct platform_device *pdev) return PTR_ERR(dcp->phy); } + if (dcp->phy) { + int ret; + + ret = phy_init(dcp->phy); + if (ret) + return dev_err_probe(dev, ret, "Failed to initialize dp-phy\n"); + + ret = phy_power_on(dcp->phy); + if (ret) { + phy_exit(dcp->phy); + return dev_err_probe(dev, ret, "Failed to power on dp-phy\n"); + } + + ret = devm_add_action_or_reset(dev, dcp_phy_power_off, dcp->phy); + if (ret) + return ret; + + /* Match the settling time used by the working cold-boot sequence. */ + msleep(25); + } bitmap_zero(dcp->iomfb_surfaces, DCP_MAX_PLANES); @@ -1254,4 +1963,8 @@ static int dcp_platform_probe(struct platform_device *pdev) return PTR_ERR(dcp->dp2hdmi_pwren); + /* The converter rails must be stable before DCP requests the link. */ + if (dcp->hdmi_pwren || dcp->dp2hdmi_pwren) + msleep(100); + ret = of_property_read_u32(dev->of_node, "mux-index", &mux_index); if (!ret) { @@ -1348,4 +2061,8 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { }; +static const struct apple_dcp_hw_data apple_dcp_hw_t8132 = { + .num_dptx_ports = 2, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, @@ -1359,4 +2076,5 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, + { .compatible = "apple,t8132-dcp", .data = &apple_dcp_hw_t8132, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 2b421772bf5d57aefc0c85a9cb3f6dd988786a48..9c131e20a8b9f1078b97fac322beb9a57503ba86 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -57,5 +57,10 @@ void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); +int dcp_dptx_phy_activate_oob(struct platform_device *pdev); int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); +void dcp_hotplug_mark_disconnected_oob(struct platform_device *pdev); +void dcp_av_disconnect_oob(struct platform_device *pdev); +int dcp_dptx_set_hpd_oob(struct platform_device *pdev, u32 port, bool hpd); +int dcp_dptx_release_oob(struct platform_device *pdev, u32 port); int iomfb_start_rtkit(struct apple_dcp *dcp); @@ -67,4 +72,6 @@ int systemep_init(struct apple_dcp *dcp); int dptxep_init(struct apple_dcp *dcp); int ibootep_init(struct apple_dcp *dcp); +int ibootep_set_surface(struct apple_dcp *dcp, dma_addr_t iova, u32 stride, + u32 width, u32 height, u32 drm_format); int dpavservep_init(struct apple_dcp *dcp); int avep_init(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 1ea00602d476d82026e3aebbeb28293a8cc95f01..48b455d9125c7e4633132fd758bddf8c19183268 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -6,4 +6,5 @@ #include #include +#include #include "afk.h" @@ -13,4 +14,14 @@ #include "trace.h" +/* + * Diagnostic switch for separating the DCP-side DPTX lifecycle from AP PHY + * MMIO. The T8132 DPTX windows are inaccessible until DCP has raised its + * internal PHY gate, so first establish that the callback transaction itself + * completes before allowing the callbacks to touch the PHY. + */ +static bool dptx_bypass_phy; +module_param_named(dptx_bypass_phy, dptx_bypass_phy, bool, 0644); +MODULE_PARM_DESC(dptx_bypass_phy, "Acknowledge DPTX PHY callbacks without MMIO"); + struct dcpdptx_connection_cmd { __le32 unk; @@ -258,4 +269,11 @@ static int dptxport_call_get_max_lane_count(struct apple_epic_service *service, return -EINVAL; + if (dptx_bypass_phy) { + dptx->lane_count = 4; + reply->retcode = cpu_to_le32(0); + reply->lane_count = cpu_to_le64(dptx->lane_count); + return 0; + } + ret = phy_validate(dptx->atcphy, PHY_MODE_DP, 0, &phy_ops); if (ret < 0) { @@ -319,5 +337,5 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic if (dptx->phy_ops.dp.set_lanes) { - if (dptx->atcphy) { + if (dptx->atcphy && !dptx_bypass_phy) { ret = phy_configure(dptx->atcphy, &dptx->phy_ops); if (ret) @@ -426,5 +444,5 @@ static int dptxport_call_set_link_rate(struct apple_epic_service *service, dptx->phy_ops.dp.set_rate = 1; - if (dptx->atcphy) { + if (dptx->atcphy && !dptx_bypass_phy) { ret = phy_configure(dptx->atcphy, &dptx->phy_ops); if (ret) @@ -478,9 +496,16 @@ dptxport_call_activate(struct apple_epic_service *service, struct dptx_port *dptx = service->cookie; const struct apple_dcp *dcp = service->ep->dcp; + int ret = 0; // TODO: hack, use phy_set_mode to select the correct DCP(EXT) input // for standalone phy (i.e. not atc phy). - if (!dcp->typec_mux) - phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + dev_info(dcp->dev, "DPTXPort: activating PHY\n"); + if (!dcp->typec_mux && !dptx_bypass_phy) { + ret = phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (ret) + dev_err(dcp->dev, "DPTXPort: PHY activation failed: %d\n", + ret); + } + dev_info(dcp->dev, "DPTXPort: PHY activation returned: %d\n", ret); memcpy(reply, data, min(reply_size, data_size)); @@ -488,5 +513,5 @@ dptxport_call_activate(struct apple_epic_service *service, memset(reply, 0, 4); - return 0; + return ret; } @@ -499,5 +524,6 @@ dptxport_call_deactivate(struct apple_epic_service *service, /* deactivate phy */ - phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); + if (!dptx_bypass_phy) + phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); memcpy(reply, data, min(reply_size, data_size)); @@ -508,9 +534,43 @@ dptxport_call_deactivate(struct apple_epic_service *service, } -static int dptxport_call(struct apple_epic_service *service, u32 idx, +/* + * Compact firmware adds a second DPTX callback group whose command numbers + * overlap the original link/PHY callbacks. Commands 3 and 4 carry an + * 80-byte record. Preserve the traced input prefix, clear the result word, + * and never echo the pointer-bearing tail back to firmware. + */ +static int dptxport_call_group1(u32 idx, const void *data, size_t data_size, + void *reply, size_t reply_size) +{ + size_t preserved = min_t(size_t, 32, min(data_size, reply_size)); + size_t result_offset; + + if (idx != 3 && idx != 4) + return -ENOSYS; + + memcpy(reply, data, preserved); + if (reply_size > preserved) + memset(reply + preserved, 0, reply_size - preserved); + + result_offset = idx == 3 ? 0 : 16; + if (reply_size < result_offset + sizeof(__le32)) + return -EINVAL; + memset(reply + result_offset, 0, sizeof(__le32)); + + return 0; +} + +static int dptxport_call(struct apple_epic_service *service, u16 group, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size) { struct dptx_port *dptx = service->cookie; + + if (group == 1) + return dptxport_call_group1(idx, data, data_size, reply, + reply_size); + if (group) + return -ENOSYS; + trace_dptxport_apcall(dptx, idx, data_size); @@ -571,9 +631,15 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { - - if (strcmp(name, "dcpdptx-port-epic")) - return; - if (strcmp(class, "AppleDCPDPTXRemotePort")) - return; + if (service->ep->compact) { + if (unit < 0) + return; + if (unit == 0) + service->defer_start = true; + } else { + if (strcmp(name, "dcpdptx-port-epic")) + return; + if (strcmp(class, "AppleDCPDPTXRemotePort")) + return; + } trace_dptxport_init(service->ep->dcp, unit); @@ -605,4 +671,9 @@ static const struct apple_epic_service_ops dptxep_ops[] = { .call = dptxport_call, }, + { + .name = "dcpdptx-port-epic", + .init = dptxport_init, + .call = dptxport_call, + }, {} }; @@ -637,4 +708,10 @@ int dptxep_init(struct apple_dcp *dcp) } + if (dcp->dptxep->compact && dcp->dptxport[0].service) { + ret = afk_start_service(dcp->dptxport[0].service); + if (ret) + return ret; + } + return 0; } diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2de9d2fe4c24a3a46cb2b7133a53100450f19545..ef972acb90d97e1905600a105131e5c268498076 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -22,8 +22,12 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { - if (dcp->dcpavserv.enabled) { - dev_err(dcp->dev, - "DCPAVSERV: unit %lld already exists\n", unit); - return; + if (dcp->dcpavserv.service && + dcp->dcpavserv.service != service) { + dev_info(dcp->dev, + "DCPAVSERV: rebinding unit %lld from interface %u to %u\n", + unit, dcp->dcpavserv.service->channel, + service->channel); + dcp->dcpavserv.service->cookie = NULL; + dcp->dcpavserv.service->enabled = false; } dcp->dcpavserv.service = service; @@ -39,5 +43,8 @@ static void dcpavserv_teardown(struct apple_epic_service *service) service->enabled = false; - if (dcp->dcpavserv.enabled) { + /* Firmware republishes this service under a new compact interface ID + * after hotplug. A delayed teardown for the old instance must not + * invalidate the replacement. */ + if (dcp->dcpavserv.service == service) { dcp->dcpavserv.enabled = false; dcp->dcpavserv.service = NULL; @@ -50,4 +57,6 @@ static void dcpdpserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { + if (service->ep->compact) + service->defer_start = true; } @@ -79,5 +88,5 @@ struct dpavserv_copy_edid_cmd { #define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E #define EDID_MAX_SIZE SZ_32K -#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) +#define EDID_COMPACT_MAX_SIZE SZ_2K struct dpavserv_copy_edid_resp { @@ -160,20 +169,34 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + size_t edid_max_size, edid_buf_size; int num_blocks; u64 data_size; int ret; + /* Compact EPIC endpoints on T8132 have a 0x1e80-byte TX ring. A + * 32-KiB inline request can never fit and fails locally with + * -EMSGSIZE. The complete 4-KiB call advances the ring by 0x1100 + * bytes, so it can also fail with -ENOMEM on an otherwise empty ring + * when the remaining tail is too short and wrapping would collide + * with the read pointer. A 2-KiB EDID limit makes the complete call + * 0x900 bytes, less than half the ring, while still accommodating 15 + * extension blocks. Retain the older maximum for non-compact + * endpoints. */ + edid_max_size = service->ep->compact ? EDID_COMPACT_MAX_SIZE : + EDID_MAX_SIZE; + edid_buf_size = EDID_LEADING_DATA_SIZE + edid_max_size; + memset(&cmd, 0, sizeof(cmd)); - cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); - resp = kzalloc(sizeof(*resp) + EDID_BUF_SIZE, GFP_KERNEL); + cmd.max_size = cpu_to_le64(edid_buf_size); + resp = kzalloc(sizeof(*resp) + edid_buf_size, GFP_KERNEL); if (!resp) return ERR_PTR(-ENOMEM); - ret = afk_service_call(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, resp, - sizeof(resp) + EDID_BUF_SIZE, 0); + ret = afk_service_call(service, 1, 7, &cmd, sizeof(cmd), edid_buf_size, + resp, sizeof(*resp) + edid_buf_size, 0); if (ret < 0) return ERR_PTR(ret); - if (le64_to_cpu(resp->max_size) != EDID_BUF_SIZE) + if (le64_to_cpu(resp->max_size) != edid_buf_size) return ERR_PTR(-EIO); @@ -182,5 +205,6 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) data_size = le64_to_cpu(resp->used_size); - if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE) + if (data_size > edid_buf_size || + data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE) return ERR_PTR(-EIO); diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index ae4bc8a69f2a8d6c0dcf82dec6b1a746b9b3d536..cce52c11a1271b757d09a4bf5b8cf7dba39459e8 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -3,11 +3,59 @@ #include +#include + +#include #include "afk.h" #include "dcp.h" +#define IBOOT_SET_SURFACE 1 +#define IBOOT_ADDR_PLANAR 1 +#define IBOOT_FMT_BGRA 1 +#define IBOOT_FMT_RGBA 3 +#define IBOOT_FMT_W30R 9 +#define IBOOT_EOTF_GAMMA_SDR 1 + +struct iboot_plane { + __le32 unk1; + __le64 addr; + __le32 tile_size; + __le32 stride; + __le32 unk2[4]; + __le32 addr_format; + __le32 unk3; +} __packed; + +struct iboot_layer { + struct iboot_plane planes[3]; + __le32 unk; + __le32 plane_cnt; + __le32 width; + __le32 height; + __le32 surface_fmt; + __le32 colorspace; + __le32 eotf; + u8 transform; + u8 padding[3]; +} __packed; + +struct iboot_set_surface_cmd { + struct iboot_layer layer; + __le32 unk3; + __le32 unk4; +} __packed; + +struct iboot_cmd { + __le32 op; + __le32 len; + __le32 unk1; + __le32 unk2; + struct iboot_set_surface_cmd payload; +} __packed; + static void disp_service_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { + service->ep->dcp->iboot_service = service; } @@ -28,2 +76,60 @@ int ibootep_init(struct apple_dcp *dcp) return 0; } + +int ibootep_set_surface(struct apple_dcp *dcp, dma_addr_t iova, u32 stride, + u32 width, u32 height, u32 drm_format) +{ + struct iboot_cmd cmd = { 0 }; + dma_addr_t surface_iova = iova & DMA_BIT_MASK(36); + u32 retcode = 0; + u32 surface_fmt; + int ret; + + if (!dcp->iboot_service) + return -ENODEV; + + switch (drm_format) { + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: + /* The retained 26.6 pipeline is configured for the bootloader's + * 10-bit packed scanout. Keep the first Linux prototype on that + * proven path; the storage remains four bytes per pixel. */ + surface_fmt = IBOOT_FMT_W30R; + break; + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_ABGR8888: + surface_fmt = IBOOT_FMT_RGBA; + break; + case DRM_FORMAT_XRGB2101010: + surface_fmt = IBOOT_FMT_W30R; + break; + default: + return -EINVAL; + } + + cmd.op = cpu_to_le32(IBOOT_SET_SURFACE); + cmd.len = cpu_to_le32(sizeof(cmd)); + /* The compact service consumes the DART-relative address. The upper + * apple,dma-range remap bits belong to the AP DMA API, not the layer. */ + cmd.payload.layer.planes[0].addr = cpu_to_le64(surface_iova); + cmd.payload.layer.planes[0].stride = cpu_to_le32(stride); + cmd.payload.layer.planes[0].addr_format = + cpu_to_le32(IBOOT_ADDR_PLANAR); + cmd.payload.layer.plane_cnt = cpu_to_le32(1); + cmd.payload.layer.width = cpu_to_le32(width); + cmd.payload.layer.height = cpu_to_le32(height); + cmd.payload.layer.surface_fmt = cpu_to_le32(surface_fmt); + /* The compact interface uses the display colorimetry enum rather than + * IOMFB's colorspace enum. Value 1 is the SDR BT.601/709 family. */ + cmd.payload.layer.colorspace = cpu_to_le32(2); + cmd.payload.layer.eotf = cpu_to_le32(IBOOT_EOTF_GAMMA_SDR); + + ret = afk_send_command(dcp->iboot_service, EPIC_SUBTYPE_STD_SERVICE, + &cmd, sizeof(cmd), NULL, 0, &retcode); + if (ret) + return ret; + if (retcode) + return -EIO; + + return 0; +} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1d9448f0f4dc479c75fd8cafa0f9b69bb3df76a7..a90b4d5b6fa7cf8801e9cb127fd75f854e69d1a8 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -9,5 +9,4 @@ #include #include -#include #include #include @@ -192,4 +191,7 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); + /* RTKit-provided firmware buffers can be mapped WC rather than allocated + * by dma_alloc_coherent(). Publish the packet before ringing DCP. */ + dma_wmb(); dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_msg(context, data_len, offset)); @@ -221,4 +223,6 @@ void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) dcp_pop_depth(&ch->depth); + /* Publish callback output before the acknowledgement makes it visible. */ + dma_wmb(); dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_ack(context)); @@ -288,6 +292,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); - if (dcp->cb_handlers[tag](dcp, tag, out, in)) + if (dcp->cb_handlers[tag](dcp, tag, out, in)) { dcp_ack(dcp, context); + } } @@ -337,4 +342,7 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) data = dcp->shmem + channel_offset + offset; + /* Pair with firmware's mailbox notification before consuming the packet + * from the retained WC mapping. */ + dma_rmb(); if (FIELD_GET(IOMFB_MSG_ACK, message)) @@ -437,4 +445,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, ret = iomfb_modeset_v13_3(dcp, crtc_state); break; + case DCP_FIRMWARE_V_26_6: + ret = iomfb_modeset_v26_6_0(dcp, crtc_state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", @@ -486,4 +497,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) iomfb_flush_v13_3(dcp, crtc, state); break; + case DCP_FIRMWARE_V_26_6: + iomfb_flush_v26_6_0(dcp, crtc, state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -501,4 +515,7 @@ static void iomfb_start(struct apple_dcp *dcp) iomfb_start_v13_3(dcp); break; + case DCP_FIRMWARE_V_26_6: + iomfb_start_v26_6_0(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -529,8 +546,14 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; - apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); - dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, - GFP_KERNEL); + if (dcp->fw_compat != DCP_FIRMWARE_V_26_6) + apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, + &shmem_iova, GFP_KERNEL); + if (!dcp->shmem) + return -ENOMEM; + + memset(dcp->shmem, 0, DCP_SHMEM_SIZE); + dma_wmb(); dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); @@ -552,4 +575,7 @@ void iomfb_shutdown(struct apple_dcp *dcp) iomfb_shutdown_v13_3(dcp); break; + case DCP_FIRMWARE_V_26_6: + iomfb_shutdown_v26_6_0(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 7903fad4040677d971c1691657e56a3e3e5ee831..c97a1ab89b78d53eeaa4ed3093f06396d6617d87 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -188,4 +188,19 @@ struct dcp_set_digital_out_mode_req { } __packed; +struct dcp_get_mode_info_req_v26_6 { + u8 mode[0x64]; + u8 mode_null; + u8 padding0[3]; + u64 opaque; + u32 timing_mode_id; + u32 color_mode_id; + u8 result_null; + u8 padding1[3]; +} __packed; + +struct dcp_get_mode_info_resp_v26_6 { + u8 opaque[0x74]; +} __packed; + struct dcp_map_buf_req { u64 buffer; @@ -291,4 +306,12 @@ struct dcp_set_power_state_req { } __packed; +struct dcp_set_power_state_req_v26_6 { + u64 state; + /* Four independent transition flags on 26.6, not a u32 boolean. */ + u8 flags[4]; + u8 result_is_null; + u8 padding[3]; +} __packed; + struct dcp_set_power_state_resp { u32 unkint; @@ -308,5 +331,5 @@ struct dcp_set_dcpav_prop_end_req { struct dcp_set_parameter_dcp { u32 param; - u32 value[8]; + u64 value[4]; u32 count; } __packed; @@ -320,4 +343,13 @@ struct dcp_swap_complete_intent_gated { } __packed; +struct dcp_swap_complete_intent_gated_v26_6 { + u32 swap_id; + u8 unkBool; + u32 unkInt; + u32 width; + u32 height; + u8 padding[3]; +} __packed; + struct dcp_read_edt_data_req { char key[0x40]; @@ -331,7 +363,28 @@ struct dcp_read_edt_data_resp { } __packed; +struct dcp_read_edt_data_resp_v26_6 { + u32 value[8]; + u8 ret; + u8 padding[3]; +} __packed; + struct iomfb_property { u32 id; u32 value; + u8 opaque[8]; +} __packed; + +struct dcp_hotplug_req_v26_6 { + u64 connected; + u8 opaque[0x50]; +} __packed; + +struct dcp_hotplug_resp_v26_6 { + u8 opaque[0x4c]; +} __packed; + +struct dcp_bool_arg { + u8 value; + u8 padding[3]; } __packed; @@ -375,4 +428,12 @@ struct iomfb_abort_swaps_dcp_resp { } __packed; +struct iomfb_abort_swaps_dcp_req_v26_6 { + u64 client; +} __packed; + +struct iomfb_abort_swaps_dcp_resp_v26_6 { + u32 ret; +} __packed; + struct iomfb_set_matrix_req { u32 location; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index cf40e273a2f43cd6576857f45e89a9edb4e5c841..2016ba875aa80a39dc0633555e1f5cfe7e234056 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -21,4 +21,5 @@ #include #include +#include #include #include @@ -26,4 +27,5 @@ #include "dcp.h" #include "dcp-internal.h" +#include "afk.h" #include "iomfb.h" #include "iomfb_internal.h" @@ -32,4 +34,106 @@ #include "version_utils.h" +enum dcp_power_transition { + DCP_POWER_TRANSITION_NONE, + DCP_POWER_TRANSITION_DISPLAY_OFF, + DCP_POWER_TRANSITION_RESTARTABLE_SLEEP, + DCP_POWER_TRANSITION_SHUTDOWN, +}; + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +typedef struct dcp_set_power_state_req_v26_6 dcp_power_state_req_t; +typedef struct iomfb_abort_swaps_dcp_req_v26_6 dcp_abort_swaps_req_t; +typedef struct iomfb_abort_swaps_dcp_resp_v26_6 dcp_abort_swaps_resp_t; +typedef struct dcp_read_edt_data_resp_v26_6 dcp_read_edt_data_resp_t; +typedef struct dcp_swap_complete_intent_gated_v26_6 dcp_swap_intent_t; +typedef struct dcp_bool_arg dcp_bool_arg_t; + +static dcp_power_state_req_t dcp_power_state_req(u64 state, + enum dcp_power_transition transition) +{ + dcp_power_state_req_t req = { + .state = state, + .padding = { 0xaa, 0xaa, 0xaa }, + }; + + switch (transition) { + case DCP_POWER_TRANSITION_RESTARTABLE_SLEEP: + req.flags[0] = 1; + req.flags[3] = 1; + req.result_is_null = 1; + break; + case DCP_POWER_TRANSITION_SHUTDOWN: + req.flags[1] = 1; + req.result_is_null = 1; + break; + case DCP_POWER_TRANSITION_NONE: + case DCP_POWER_TRANSITION_DISPLAY_OFF: + break; + } + + return req; +} + +static dcp_bool_arg_t dcp_bool_arg(bool value) +{ + return (dcp_bool_arg_t){ + .value = value, + .padding = { 0xaa, 0xaa, 0xaa }, + }; +} +#else +typedef struct dcp_set_power_state_req dcp_power_state_req_t; +typedef struct iomfb_abort_swaps_dcp_req dcp_abort_swaps_req_t; +typedef struct iomfb_abort_swaps_dcp_resp dcp_abort_swaps_resp_t; +typedef struct dcp_read_edt_data_resp dcp_read_edt_data_resp_t; +typedef struct dcp_swap_complete_intent_gated dcp_swap_intent_t; +typedef u32 dcp_bool_arg_t; + +static dcp_power_state_req_t dcp_power_state_req(u64 state, + enum dcp_power_transition transition) +{ + return (dcp_power_state_req_t){ + .unklong = state, + .unkbool = transition != DCP_POWER_TRANSITION_NONE, + .unkint_null = transition != DCP_POWER_TRANSITION_NONE, + }; +} + +static dcp_bool_arg_t dcp_bool_arg(bool value) +{ + return value; +} +#endif + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +static u64 dcp_iomfb_client(struct apple_dcp *dcp) +{ + /* Native passes its IOUserClient pointer as an opaque client identity. */ + return (u64)(unsigned long)dcp; +} +#endif + +static struct DCP_FW_NAME(dcp_swap_start_req) +dcp_swap_start_request(struct apple_dcp *dcp) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + return (struct DCP_FW_NAME(dcp_swap_start_req)){ + .client = dcp_iomfb_client(dcp), + .padding = { 0xaa, 0xaa, 0xaa }, + }; +#else + return (struct DCP_FW_NAME(dcp_swap_start_req)){ 0 }; +#endif +} + +static dcp_abort_swaps_req_t dcp_abort_swaps_request(struct apple_dcp *dcp) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + return (dcp_abort_swaps_req_t){ .client = dcp_iomfb_client(dcp) }; +#else + return (dcp_abort_swaps_req_t){ .client.flag2 = 1 }; +#endif +} + /* Register defines used in bandwidth setup structure */ #define REG_DOORBELL_BIT(idx) (2 + (idx)) @@ -55,5 +159,6 @@ IOMFB_THUNK_INOUT(set_matrix); IOMFB_THUNK_INOUT(get_color_remap_mode); IOMFB_THUNK_INOUT(last_client_close); -IOMFB_THUNK_INOUT(abort_swaps_dcp); +DCP_THUNK_INOUT(iomfb_abort_swaps_dcp, iomfbep_abort_swaps_dcp, + dcp_abort_swaps_req_t, dcp_abort_swaps_resp_t); DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, @@ -61,9 +166,10 @@ DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_resp)); -DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, - struct dcp_swap_start_resp); +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, + struct DCP_FW_NAME(dcp_swap_start_req), + struct DCP_FW_NAME(dcp_swap_start_resp)); DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, - struct dcp_set_power_state_req, + dcp_power_state_req_t, struct dcp_set_power_state_resp); @@ -77,9 +183,11 @@ DCP_THUNK_OUT(dcp_set_display_refresh_properties, #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -DCP_THUNK_INOUT(dcp_late_init_signal, dcpep_late_init_signal, u32, u32); +DCP_THUNK_INOUT(dcp_late_init_signal, dcpep_late_init_signal, + dcp_bool_arg_t, u32); #else DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); #endif -DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); +DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, + dcp_bool_arg_t); DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); @@ -262,6 +370,6 @@ static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_pr * stream of the display DART, rather than the expected DCP DART. */ -static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, - struct dcp_map_buf_req *req) +static struct DCP_FW_NAME(dcp_map_buf_resp) +dcpep_cb_map_piodma(struct apple_dcp *dcp, struct dcp_map_buf_req *req) { struct dcp_mem_descriptor *memdesc; @@ -290,10 +398,24 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, } - return (struct dcp_map_buf_resp){ .dva = memdesc->dva }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = memdesc->dva, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = memdesc->dva }; +#endif reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); - return (struct dcp_map_buf_resp){ .ret = EINVAL }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .ret = EINVAL, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .ret = EINVAL }; +#endif } @@ -472,7 +594,23 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d struct resource *rsrc = dcp->disp_registers[req->index]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), - DMA_BIDIRECTIONAL, 0); + dma_addr_t dva; + size_t size = resource_size(rsrc); + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + /* + * 26.6 returns register-window physical addresses directly. DCP + * issues these firmware-owned MMIO transactions on its dedicated + * DART-bypass stream; installing a mapping in the normal DMA stream + * does not affect them. + */ + dva = rsrc->start; + dev_dbg(dcp->dev, + "D411: physical reg[%u] DVA/PA %#llx size %#zx\n", + req->index, (unsigned long long)dva, size); +#else + dva = dma_map_resource(dcp->dev, rsrc->start, size, + DMA_BIDIRECTIONAL, 0); WARN_ON(dva == DMA_MAPPING_ERROR); +#endif #endif @@ -487,8 +625,8 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d } -static struct dcp_read_edt_data_resp +static dcp_read_edt_data_resp_t dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) { - return (struct dcp_read_edt_data_resp){ + return (dcp_read_edt_data_resp_t){ .value[0] = req->value[0], .ret = 0, @@ -625,5 +763,5 @@ static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u32 v_true = 1; + dcp_bool_arg_t v_true = dcp_bool_arg(true); dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); #else @@ -634,5 +772,5 @@ static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) { - u32 v_true = true; + dcp_bool_arg_t v_true = dcp_bool_arg(true); dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); @@ -675,5 +813,5 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) }; - if (dcp->disp_bw_scratch_index) { + if (!dcp_disable_rt_bandwidth && dcp->disp_bw_scratch_index) { u32 offset = dcp->disp_bw_scratch_offset; u32 index = dcp->disp_bw_scratch_index; @@ -681,5 +819,5 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) } - if (dcp->disp_bw_doorbell_index) { + if (!dcp_disable_rt_bandwidth && dcp->disp_bw_doorbell_index) { u32 index = dcp->disp_bw_doorbell_index; rt_bw.reg_doorbell = dcp->disp_registers[index]->start; @@ -696,4 +834,17 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +static struct dcp_set_frame_sync_props_resp_v26_6_0 +dcpep_cb_set_frame_sync_props( + struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req_v26_6_0 *req) +{ + struct dcp_set_frame_sync_props_resp_v26_6_0 resp; + + memcpy(resp.props, req->props, sizeof(resp.props)); + memset(&resp.props[73], 0, 3); + return resp; +} +#else static struct dcp_set_frame_sync_props_resp dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, @@ -702,4 +853,5 @@ dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, return (struct dcp_set_frame_sync_props_resp){}; } +#endif /* Callback to get the current time as milliseconds since the UNIX epoch */ @@ -755,5 +907,5 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_resp *resp = data; + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; @@ -776,33 +928,105 @@ static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) } +static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, + void *cookie); + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +static const struct dcp_method_entry dcp_prepare_wake_method = { + .name = "dcpep_prepare_wake", + .tag = { 'A', '4', '1', '4' }, +}; + +static const struct dcp_method_entry dcp_finish_wake_method = { + .name = "dcpep_finish_wake", + .tag = { 'A', '4', '7', '4' }, +}; + +static const struct dcp_method_entry dcp_get_mode_info_method = { + .name = "dcpep_get_mode_info", + .tag = { 'A', '3', '6', '4' }, +}; +#endif + static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req req = { - .unklong = 1, - }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + u32 handle = dcp->main_display ? 0 : 2; + + dcp_set_display_device(dcp, false, &handle, + dcp->main_display ? dcp_on_final : dcp_on_set_parameter, + cookie); +#else + dcp_power_state_req_t req = + dcp_power_state_req(1, DCP_POWER_TRANSITION_NONE); dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); +#endif } static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, - .value = { 0 }, + struct dcp_set_parameter_dcp param; + u32 count = #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - .count = 3, + 3; #else - .count = 1, + 1; #endif - }; - dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_power_state, cookie); + memset(¶m, 0xaa, sizeof(param)); + param.param = IOMFBPARAM_ADAPTIVE_SYNC; + memset(param.value, 0, count * sizeof(param.value[0])); + param.count = count; + + dcp_set_parameter_dcp(dcp, false, ¶m, +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + dcp_on_final, +#else + dcp_on_set_power_state, +#endif + cookie); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +static void dcp_poweron_after_main_display(struct apple_dcp *dcp, void *out, + void *cookie) +{ + dcp_power_state_req_t power_req = + dcp_power_state_req(1, DCP_POWER_TRANSITION_NONE); + + dcp_set_power_state(dcp, false, &power_req, dcp_on_set_power_state, + cookie); +} + +static void dcp_poweron_after_finish_wake(struct apple_dcp *dcp, void *out, + void *cookie) +{ + dcp_is_main_display(dcp, false, dcp_poweron_after_main_display, cookie); +} + +static void dcp_poweron_after_first_client(struct apple_dcp *dcp, void *out, + void *cookie) +{ + dcp_push(dcp, false, &dcp_finish_wake_method, 0, sizeof(u32), NULL, + dcp_poweron_after_finish_wake, cookie); +} + +static void dcp_poweron_after_prepare_wake(struct apple_dcp *dcp, void *out, + void *cookie) +{ + dcp_first_client_open(dcp, false, dcp_poweron_after_first_client, + cookie); +} +#endif + void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) { struct dcp_wait_cookie *cookie; int ret; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + const u8 prepare_wake[8] = { 0, 0, 0, 0, 0, 0xaa, 0xaa, 0xaa }; +#else u32 handle; +#endif dev_info(dcp->dev, "dcp_poweron() starting\n"); @@ -816,4 +1040,13 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) kref_get(&cookie->refcount); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + /* This is the wake ordering observed on 26.6 after A500 put the + * external-display pipeline to sleep. Omitting A414/A474, or selecting + * the display before A473, leaves fControllerPowerState at zero and DCP + * silently swallows every scanout swap. */ + dcp_push(dcp, false, &dcp_prepare_wake_method, sizeof(prepare_wake), + sizeof(u64), (void *)prepare_wake, + dcp_poweron_after_prepare_wake, cookie); +#else if (dcp->main_display) { handle = 0; @@ -825,4 +1058,5 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) dcp_on_set_parameter, cookie); } +#endif ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(10000)); @@ -858,9 +1092,28 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, } -static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +static void dcp_poweroff_after_main_display(struct apple_dcp *dcp, void *out, + void *cookie) { - struct dcp_set_power_state_req power_req = { - .unklong = 0, - }; + dcp_power_state_req_t power_req = + dcp_power_state_req(0, DCP_POWER_TRANSITION_DISPLAY_OFF); + + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, + cookie); +} + +static void dcp_poweroff_after_prepare_state(struct apple_dcp *dcp, void *out, + void *cookie) +{ + dcp_is_main_display(dcp, false, dcp_poweroff_after_main_display, cookie); +} +#endif + +#if DCP_FW_VER < DCP_FW_VERSION(26, 6, 0) +static void last_client_closed_poff(struct apple_dcp *dcp, void *out, + void *cookie) +{ + dcp_power_state_req_t power_req = + dcp_power_state_req(0, DCP_POWER_TRANSITION_DISPLAY_OFF); dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, cookie); @@ -873,16 +1126,16 @@ static void aborted_swaps_dcp_poff(struct apple_dcp *dcp, void *out, void *cooki last_client_closed_poff, cookie); } +#endif void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) { int ret, swap_id; - struct iomfb_abort_swaps_dcp_req abort_req = { - .client = { - .flag2 = 1, - }, - }; +#if DCP_FW_VER < DCP_FW_VERSION(26, 6, 0) + dcp_abort_swaps_req_t abort_req = dcp_abort_swaps_request(dcp); +#endif struct dcp_swap_cookie *cookie; struct dcp_wait_cookie *poff_cookie; - struct dcp_swap_start_req swap_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_start_req) swap_req = + dcp_swap_start_request(dcp); struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); @@ -910,5 +1163,9 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) */ if (dcp_has_panel(dcp)) { +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) + swap->swap.bl_update = 1; +#else swap->swap.bl_unk = 1; +#endif swap->swap.bl_value = 0; swap->swap.bl_power = 0; @@ -919,5 +1176,5 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) swap->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - for (int l = 0; l < 5; l++) + for (int l = 0; l < ARRAY_SIZE(swap->surf2_null); l++) swap->surf2_null[l] = true; swap->unkU32Ptr_null = true; @@ -945,6 +1202,22 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) kref_get(&poff_cookie->refcount); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + /* Native 26.6 ordinary hot-unplug goes directly from its clear + * transaction through A414/A412/A473. A469 abort-swaps and A458 + * last-client-close only appear in full shutdown/sleep traces, and may + * tear down state which the link power-off callbacks still require. */ + { + const u8 prepare_state[8] = { + 0, 0, 0, 0, 0, 0xaa, 0xaa, 0xaa, + }; + + dcp_push(dcp, false, &dcp_prepare_wake_method, + sizeof(prepare_state), sizeof(u64), (void *)prepare_state, + dcp_poweroff_after_prepare_state, poff_cookie); + } +#else iomfb_abort_swaps_dcp(dcp, false, &abort_req, aborted_swaps_dcp_poff, poff_cookie); +#endif ret = wait_for_completion_timeout(&poff_cookie->done, msecs_to_jiffies(1000)); @@ -964,7 +1237,6 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req power_req = { - .unklong = 0, - }; + dcp_power_state_req_t power_req = + dcp_power_state_req(0, DCP_POWER_TRANSITION_RESTARTABLE_SLEEP); dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, cookie); } @@ -979,9 +1251,5 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) { int ret; - struct iomfb_abort_swaps_dcp_req req = { - .client = { - .flag2 = 1, - }, - }; + dcp_abort_swaps_req_t req = dcp_abort_swaps_request(dcp); struct dcp_wait_cookie *cookie; @@ -1007,5 +1275,5 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) } -static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) +static void dcpep_cb_hotplug_state(struct apple_dcp *dcp, u64 connected) { struct apple_connector *connector = dcp->connector; @@ -1024,13 +1292,13 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) dev_info(dcp->dev, "cb_hotplug() ignored during modeset connected:%llu\n", - *connected); + connected); return; } dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d\n", - *connected, dcp->valid_mode); + connected, dcp->valid_mode); /* Hotplug invalidates mode. DRM doesn't always handle this. */ - if (!(*connected)) { + if (!connected) { dcp->valid_mode = false; /* after unplug swap will not complete until the next @@ -1039,6 +1307,6 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) } - if (connector && connector->connected != !!(*connected)) { - connector->connected = !!(*connected); + if (connector && connector->connected != !!connected) { + connector->connected = !!connected; dcp->valid_mode = false; schedule_work(&connector->hotplug_wq); @@ -1046,7 +1314,31 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +static struct dcp_hotplug_resp_v26_6 +dcpep_cb_hotplug(struct apple_dcp *dcp, struct dcp_hotplug_req_v26_6 *req) +{ + struct dcp_hotplug_resp_v26_6 resp = { 0 }; + + dcpep_cb_hotplug_state(dcp, req->connected); + + /* Every captured 26.6 connected response has an unaligned 0xffffffff + * field at offsets 71..74. Byte 75 is padding and varies in native + * traces, so leave it initialized to zero. The disconnected response + * contains additional state that is not decoded yet. */ + if (req->connected) + memset(&resp.opaque[0x47], 0xff, sizeof(u32)); + + return resp; +} +#else +static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) +{ + dcpep_cb_hotplug_state(dcp, *connected); +} +#endif + static void dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, - struct dcp_swap_complete_intent_gated *info) + dcp_swap_intent_t *info) { trace_iomfb_swap_complete_intent_gated(dcp, info->swap_id, @@ -1086,5 +1378,6 @@ TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, struct iomfb_set_fx_prop_req) TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, - struct dcp_map_buf_req, struct dcp_map_buf_resp); + struct dcp_map_buf_req, + struct DCP_FW_NAME(dcp_map_buf_resp)); TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, struct dcp_unmap_buf_resp); @@ -1102,5 +1395,5 @@ TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, struct DCP_FW_NAME(dcp_map_reg_resp)); TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, - struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); + struct dcp_read_edt_data_req, dcp_read_edt_data_resp_t); TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, @@ -1112,13 +1405,26 @@ TRAMPOLINE_INOUT(trampoline_allocate_bandwidth, dcpep_cb_allocate_bandwidth, TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, struct dcp_rt_bandwidth); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, + dcpep_cb_set_frame_sync_props, + struct dcp_set_frame_sync_props_req_v26_6_0, + struct dcp_set_frame_sync_props_resp_v26_6_0); +#else TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, - struct dcp_set_frame_sync_props_req, - struct dcp_set_frame_sync_props_resp); + struct dcp_set_frame_sync_props_req, + struct dcp_set_frame_sync_props_resp); +#endif TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +TRAMPOLINE_INOUT(trampoline_hotplug, dcpep_cb_hotplug, + struct dcp_hotplug_req_v26_6, + struct dcp_hotplug_resp_v26_6); +#else TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); +#endif TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, dcpep_cb_swap_complete_intent_gated, - struct dcp_swap_complete_intent_gated); + dcp_swap_intent_t); TRAMPOLINE_IN(trampoline_abort_swap_ap_gated, dcpep_cb_abort_swap_ap_gated, u32); TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, @@ -1161,5 +1467,5 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_resp *resp = data; + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; @@ -1172,5 +1478,6 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { - struct dcp_swap_start_req start_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_start_req) start_req = + dcp_swap_start_request(dcp); if (dcp->connector && dcp->connector->connected) @@ -1191,4 +1498,60 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +struct dcp_modeset_cookie { + /* Must be first: release_wait_cookie() frees this allocation. */ + struct dcp_wait_cookie wait; + struct dcp_set_digital_out_mode_req actual; + struct dcp_set_digital_out_mode_req bootstrap; +}; + +static void dcp_get_mode_info(struct apple_dcp *dcp, + const struct dcp_set_digital_out_mode_req *mode, + dcp_callback_t cb, void *cookie) +{ + struct dcp_get_mode_info_req_v26_6 req; + + /* Match the initialized bytes in native 26.6 requests. A364 fills the + * mode record before A413 consumes the selected timing/color pair. */ + memset(&req, 0xaa, sizeof(req)); + memset(req.mode, 0, sizeof(req.mode)); + req.mode_null = 0; + req.timing_mode_id = mode->timing_mode_id; + req.color_mode_id = mode->color_mode_id; + req.result_null = 0; + memset(req.padding1, 0, sizeof(req.padding1)); + + dcp_push(dcp, false, &dcp_get_mode_info_method, sizeof(req), + sizeof(struct dcp_get_mode_info_resp_v26_6), &req, cb, cookie); +} + +static void dcp_modeset_after_actual_info(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_modeset_cookie *modeset = cookie; + + dcp_set_digital_out_mode(dcp, false, &modeset->actual, + complete_set_digital_out_mode, cookie); +} + +static void dcp_modeset_after_bootstrap(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_modeset_cookie *modeset = cookie; + + dcp_get_mode_info(dcp, &modeset->actual, + dcp_modeset_after_actual_info, cookie); +} + +static void dcp_modeset_after_bootstrap_info(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_modeset_cookie *modeset = cookie; + + dcp_set_digital_out_mode(dcp, false, &modeset->bootstrap, + dcp_modeset_after_bootstrap, cookie); +} +#endif + int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) @@ -1197,4 +1560,7 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, struct dcp_wait_cookie *cookie; struct dcp_color_mode *cmode = NULL; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + struct dcp_modeset_cookie *modeset_cookie; +#endif int ret; @@ -1233,5 +1599,13 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, dcp->use_timestamps = mode->vrr; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + modeset_cookie = kzalloc(sizeof(*modeset_cookie), GFP_KERNEL); + if (!modeset_cookie) + return -ENOMEM; + cookie = &modeset_cookie->wait; + modeset_cookie->actual = dcp->mode; +#else cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); +#endif if (!cookie) { return -ENOMEM; @@ -1245,6 +1619,27 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, dcp->during_modeset = true; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + if (!dcp->valid_mode && mode->bootstrap.score >= 0) { + modeset_cookie->bootstrap = + (struct dcp_set_digital_out_mode_req) { + .color_mode_id = mode->bootstrap.id, + .timing_mode_id = mode->timing_mode_id, + }; + dev_info(dcp->dev, + "bootstrapping generic pipe (color:%u timing:%u)\n", + modeset_cookie->bootstrap.color_mode_id, + modeset_cookie->bootstrap.timing_mode_id); + dcp_get_mode_info(dcp, &modeset_cookie->bootstrap, + dcp_modeset_after_bootstrap_info, + modeset_cookie); + } else { + dcp_get_mode_info(dcp, &modeset_cookie->actual, + dcp_modeset_after_actual_info, + modeset_cookie); + } +#else dcp_set_digital_out_mode(dcp, false, &dcp->mode, complete_set_digital_out_mode, cookie); +#endif /* @@ -1294,9 +1689,13 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->surf_null[l] = true; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - for (l = 0; l < 5; l++) + for (l = 0; l < ARRAY_SIZE(req->surf2_null); l++) req->surf2_null[l] = true; req->unkU32Ptr_null = true; req->unkU32out_null = true; #endif +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + req->swap.presents_surface = 1; + req->swap.has_completion = 1; +#endif /* @@ -1342,10 +1741,9 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru } - if (!new_state->fb || !new_state->visible) { + if (!new_state->fb || !new_state->visible) continue; - } + req->surf_null[l] = false; has_surface = 1; - req->swap.src_rect[l] = apple_state->src_rect; req->swap.dst_rect[l] = apple_state->dst_rect; @@ -1357,9 +1755,4 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->surf[l].base = apple_state->surf; - /* Use sRGB colorspace only for internal panels. External - * displays are expected to have EDID and user space can use - * the contained colorimetry information to provide native - * colors. - */ if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP && req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) @@ -1395,5 +1788,9 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru /* update brightness if changed */ if (dcp_has_panel(dcp) && dcp->brightness.update) { +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) + req->swap.bl_update = 1; +#else req->swap.bl_unk = 1; +#endif req->swap.bl_value = dcp->brightness.dac; req->swap.bl_power = 0x40; @@ -1466,7 +1863,6 @@ static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) { - struct dcp_set_power_state_req req = { - /* defaults are ok */ - }; + dcp_power_state_req_t req = + dcp_power_state_req(0, DCP_POWER_TRANSITION_SHUTDOWN); dcp_set_power_state(dcp, false, &req, NULL, NULL); diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 8efab49cc53d08964a5da5476fb99c050923aaa2..0021b7dac3af2de7245b566dd40c096d0d7c66b9 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -32,4 +32,8 @@ struct DCP_FW_NAME(dcp_swap) { u64 flags2; +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) + u8 unk_v14_7[0x48]; +#endif + u32 swap_id; @@ -43,8 +47,16 @@ struct DCP_FW_NAME(dcp_swap) { u32 bg_color; - u8 unk_110[0x1b8]; + u8 unk_110[0x30]; + u32 active_region_en[SWAP_SURFACES]; + struct dcp_rect active_regions[SWAP_SURFACES]; + u8 unk_190[0x138]; u32 unk_2c8; +#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) u8 unk_2cc[0x14]; u32 unk_2e0; +#else + u8 unk_2cc[0x40]; + u32 bl_update; +#endif #if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) u16 unk_2e2; @@ -52,11 +64,27 @@ struct DCP_FW_NAME(dcp_swap) { u8 unk_2e2[3]; #endif +#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) u64 bl_unk; +#else + u32 bl_unk; +#endif u32 bl_value; // min value is 0x10000000 u8 bl_power; // constant 0x40 for on u8 unk_2f3[0x2d]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u8 unk_320[0x13f]; - u64 unk_1; + u8 unk_320[0x147]; +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) + u8 unk_14_7_2[0x30]; +#endif +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + /* Added after the 14.7 swap record. Native 26.6 sets both bytes for + * ordinary visible swaps; the remaining bytes have always been zero in + * traces. */ + u8 unk_26_6[0x16]; + u8 presents_surface; + u8 unk_26_6_17[0x51]; + u8 has_completion; + u8 unk_26_6_69[0x17]; +#endif #endif } __packed; @@ -80,6 +108,11 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u64 unk_u64_a[SWAP_SURFACES]; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + struct DCP_FW_NAME(dcp_surface) surf2[6]; + u64 surf2_iova[6]; +#else struct DCP_FW_NAME(dcp_surface) surf2[5]; u64 surf2_iova[5]; +#endif #endif u8 unkbool; @@ -96,5 +129,9 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u8 surf_null[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + u8 surf2_null[6]; +#else u8 surf2_null[5]; +#endif #endif u8 unkoutbool_null; @@ -103,5 +140,14 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u8 unkU32out_null; #endif +#if DCP_FW_VER < DCP_FW_VERSION(26, 6, 0) u8 padding[1]; +#endif +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) && \ + DCP_FW_VER < DCP_FW_VERSION(26, 6, 0) + u8 padding_14_7[0x1e9]; + u8 unk_14_7_zero[0x46]; + u32 unk_14_7_u32; + u8 unk_bool; +#endif } __packed; @@ -117,4 +163,7 @@ struct DCP_FW_NAME(dcp_swap_submit_resp) { struct DCP_FW_NAME(dc_swap_complete_resp) { u32 swap_id; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) + u8 opaque[0x72c]; +#else u8 unkbool; u64 swap_data; @@ -126,6 +175,47 @@ struct DCP_FW_NAME(dc_swap_complete_resp) { u32 unkint; u8 swap_info_null; +#endif } __packed; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 6, 0) +struct DCP_FW_NAME(dcp_swap_start_req) { + u32 swap_id; + u64 client; + u8 swap_id_null; + u8 padding[3]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { + u32 swap_id; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_map_buf_resp) { + u32 buffer; + u64 dva; + u32 ret; +} __packed; +#else +struct DCP_FW_NAME(dcp_swap_start_req) { + u32 swap_id; + struct dcp_iouserclient client; + u8 swap_id_null; + u8 client_null; + u8 padding[2]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { + u32 swap_id; + struct dcp_iouserclient client; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_map_buf_resp) { + u64 vaddr; + u64 dva; + u32 ret; +} __packed; +#endif + struct DCP_FW_NAME(dcp_map_reg_req) { char obj[4]; diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index 0fe08c42d646594802c89cf5665b18a0eb2f5c63..f7592dffd9c01953436f8cc26c78d2495b4d09eb 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -4,4 +4,5 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" +#include "iomfb_v26_6.h" #include "version_utils.h" diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c index 0ac869d24eb01b5f7770ccda4429f8db78360701..044e1bc673bd00d8e6059664dd4682ad45bb602e 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_3.c +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -4,4 +4,5 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" +#include "iomfb_v26_6.h" #include "version_utils.h" diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.c b/drivers/gpu/drm/apple/iomfb_v26_6.c new file mode 100644 index 0000000000000000000000000000000000000000..488e4a88a97dbee4e5c9ffae163246afde016304 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v26_6.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include + +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "iomfb_v26_6.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A029", dcpep_setup_video_limits), + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A385", dcpep_set_create_dfb), + IOMFB_METHOD("A386", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + IOMFB_METHOD("A407", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A442", dcpep_set_parameter_dcp), + IOMFB_METHOD("A446", dcpep_create_default_fb), + IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A457", dcpep_first_client_open), + IOMFB_METHOD("A458", iomfbep_last_client_close), + IOMFB_METHOD("A465", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A468", dcpep_flush_supports_power), + IOMFB_METHOD("A469", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A473", dcpep_set_power_state), +}; + +#define DCP_FW v26_6_0 +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.c" + +struct dcp_dict_passthrough_req_v26_6 { + u8 key[0x40]; + u8 value[0x1000]; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcp_dict_passthrough_resp_v26_6 { + u8 value[0x1000]; + u8 ret; + u8 padding[3]; +} __packed; + +/* + * D114 periodically publishes dictionary-valued DCP statistics. The value + * is an in/out pointer: native macOS returns the complete serialized buffer + * byte-for-byte and reports false. Returning an all-zero output is not a + * harmless false result because DCP deserializes the in/out value first. + */ +static bool trampoline_passthrough_dict_v26_6(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_dict_passthrough_req_v26_6 *req = in; + struct dcp_dict_passthrough_resp_v26_6 *resp = out; + + trace_iomfb_callback(dcp, tag, "dcpep_cb_passthrough_dict"); + + if (!req->value_null) + memcpy(resp->value, req->value, sizeof(resp->value)); + + resp->ret = false; + memset(resp->padding, 0, sizeof(resp->padding)); + + return true; +} + +static_assert(sizeof(struct dcp_swap_v26_6_0) == 0x588); +static_assert(sizeof(struct dcp_swap_start_req_v26_6_0) == 0x10); +static_assert(sizeof(struct dcp_swap_start_resp_v26_6_0) == 0x8); +static_assert(sizeof(struct dcp_swap_submit_req_v26_6_0) == 0x1bd8); +static_assert(sizeof(struct dcp_swap_submit_resp_v26_6_0) == 0xc); +static_assert(sizeof(struct dc_swap_complete_resp_v26_6_0) == 0x730); +static_assert(sizeof(struct dcp_map_buf_resp_v26_6_0) == 0x10); +static_assert(sizeof(struct dcp_set_power_state_req_v26_6) == 0x10); +static_assert(sizeof(struct dcp_set_parameter_dcp) == 0x28); +static_assert(sizeof(struct dcp_get_mode_info_req_v26_6) == 0x7c); +static_assert(sizeof(struct dcp_get_mode_info_resp_v26_6) == 0x74); +static_assert(sizeof(struct iomfb_abort_swaps_dcp_req_v26_6) == 0x8); +static_assert(sizeof(struct iomfb_abort_swaps_dcp_resp_v26_6) == 0x4); +static_assert(sizeof(struct dcp_swap_complete_intent_gated_v26_6) == 0x14); +static_assert(sizeof(struct dcp_hotplug_req_v26_6) == 0x58); +static_assert(sizeof(struct dcp_hotplug_resp_v26_6) == 0x4c); +static_assert(sizeof(struct iomfb_property) == 0x10); +static_assert(sizeof(struct dcp_read_edt_data_resp_v26_6) == 0x24); +static_assert(sizeof(struct dcp_bool_arg) == 0x4); +static_assert(sizeof(struct dcp_set_frame_sync_props_req_v26_6_0) == 0x54); +static_assert(sizeof(struct dcp_set_frame_sync_props_resp_v26_6_0) == 0x50); +static_assert(sizeof(struct dcp_dict_passthrough_req_v26_6) == 0x1044); +static_assert(sizeof(struct dcp_dict_passthrough_resp_v26_6) == 0x1004); + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, + [1] = trampoline_true, + [2] = trampoline_nop, + [3] = trampoline_rt_bandwidth, + [6] = trampoline_set_frame_sync_props, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, + [102] = trampoline_nop, + [104] = trampoline_nop, + [107] = trampoline_nop, + [108] = trampoline_true, + [109] = trampoline_true, + [110] = trampoline_true, + [111] = trampoline_true, + [112] = trampoline_create_backlight_service, + [113] = trampoline_true, + [114] = trampoline_passthrough_dict_v26_6, + /* set_tiling_state: 26.6 returns the value followed by success. */ + [115] = trampoline_get_tiling_state, + [121] = dcpep_cb_boot_1, + [123] = trampoline_false, + [125] = trampoline_read_edt_data, + [127] = trampoline_prop_start, + [128] = trampoline_prop_chunk, + [129] = trampoline_prop_end, + [134] = trampoline_zero, + [201] = trampoline_map_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_nop, + [209] = trampoline_get_time, + [300] = trampoline_pr_publish, + [400] = trampoline_nop, + [401] = trampoline_get_uint_prop, + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, + [451] = trampoline_allocate_buffer, + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + /* Native 26.6 returns a zero status for both start and stop events. */ + [572] = trampoline_zero, + [575] = trampoline_hotplug, + [576] = trampoline_nop, + [581] = trampoline_nop, + [590] = trampoline_swap_complete, + [592] = trampoline_swap_complete_intent_gated, + [595] = trampoline_nop, + [598] = trampoline_nop, + [599] = trampoline_nop, +}; + +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.h b/drivers/gpu/drm/apple/iomfb_v26_6.h new file mode 100644 index 0000000000000000000000000000000000000000..8f373ab4102cc141b6042fc32fa24b7a24c8138e --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v26_6.h @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_V26_6_H__ +#define __APPLE_IOMFB_V26_6_H__ + +#include "version_utils.h" + +#define DCP_FW v26_6_0 +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.h" + +#undef DCP_FW_VER +#undef DCP_FW + +/* D006 set_frame_sync_props grew from 32/28 bytes to 84/80 bytes in 26.6. + * Native copies the first 80 request bytes into the response and clears the + * three flag bytes at offsets 73..75. */ +struct dcp_set_frame_sync_props_req_v26_6_0 { + u8 props[0x50]; + u8 opaque[0x4]; +} __packed; + +struct dcp_set_frame_sync_props_resp_v26_6_0 { + u8 props[0x50]; +} __packed; + +#endif /* __APPLE_IOMFB_V26_6_H__ */ diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 56960fa1fda2658668045026860bf2ab7c30fc0d..d0b20f2d203534051b0d28e51fe3adcf579a2a28 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -367,9 +367,18 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, out->sdr.score = -1; out->best.score = -1; + out->bootstrap.score = -1; dcp_parse_foreach_in_array(handle, outer_it) { struct iterator it; bool is_virtual = true; - struct color_mode cmode; + struct color_mode cmode = { + .score = -1, + .id = -1, + .depth = -1, + .pixel_encoding = -1, + .dynamic_range = -1, + .colorimetry = -1, + .eotf = -1, + }; dcp_parse_foreach_in_dict(handle, it) { @@ -404,6 +413,7 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, } - /* Skip virtual or partial entries */ - if (is_virtual || cmode.score < 0 || cmode.id < 0) + /* Skip partial entries. 26.6 uses a virtual color element to + * bootstrap the generic pipe before selecting a real color mode. */ + if (cmode.score < 0 || cmode.id < 0) continue; @@ -413,4 +423,9 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.pixel_encoding); + if (is_virtual) { + fill_color_mode(&out->bootstrap, &cmode); + continue; + } + if (cmode.eotf == DCP_EOTF_SDR_GAMMA) { if (cmode.pixel_encoding == DCP_COLOR_FORMAT_RGB && diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 52c4de46d2f79f83bb80da84cc148ea368236596..c5fc83c61fb74957278d2f983706d7600b681a0a 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -94,4 +94,6 @@ struct dcp_display_mode { struct dcp_color_mode sdr; struct dcp_color_mode best; + /* Virtual color element used to bootstrap the generic display pipe. */ + struct dcp_color_mode bootstrap; bool vrr; }; diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 51a6c2dbd7a399fad3235a97fda1126eb0e40228..6a4cea28db2172f03c45ac44fc922e2c9415bb84 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -334,4 +334,7 @@ apple_plane_duplicate_state(struct drm_plane *plane) apple_plane_state->surf = old_apple_plane_state->surf; + apple_plane_state->src_rect = old_apple_plane_state->src_rect; + apple_plane_state->dst_rect = old_apple_plane_state->dst_rect; + apple_plane_state->iova = old_apple_plane_state->iova; return &apple_plane_state->base; diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index dfc8987d2ced36353d03edcb4a55f8b48257e9f9..daf29ca2864663b30278caf064825a474a39098b 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -1,4 +1,19 @@ # SPDX-License-Identifier: GPL-2.0 +config DRM_ASAHI_M4 + tristate "Apple M4 GPU driver" + depends on RUST && DRM=y + depends on ARM64 && ARCH_APPLE && PAGE_SIZE_16KB + depends on PM + select RUST_DRM_GEM_SHMEM_HELPER + select RUST_DRM_SCHED + select RUST_APPLE_RTKIT + select APPLE_MAILBOX + help + Rust DRM driver for the T8132 (M4) GPU, following the Asahi UAPI + and the M4 Python DRM-shim firmware implementation. Intended for + development under m1n1. Normal initialization is the default; + probe_only selects identification without firmware startup. + config RUST_DRM_SCHED bool diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index e67248667987607aecc7092bcfe8a22ba524dbbd..b0d5951b26e6dc162fa8273c4b11836e2b63296a 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -2,2 +2,3 @@ obj-$(CONFIG_DRM_ASAHI) += asahi.o +obj-$(CONFIG_DRM_ASAHI_M4) += asahi_m4.o diff --git a/drivers/gpu/drm/asahi/asahi_m4.rs b/drivers/gpu/drm/asahi/asahi_m4.rs new file mode 100644 index 0000000000000000000000000000000000000000..94c21acb6e7dd07a197def39c86a37436b488a83 --- /dev/null +++ b/drivers/gpu/drm/asahi/asahi_m4.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors + +//! T8132 GPU driver, ported from the working Python DRM shim. + +#![recursion_limit = "2048"] + +mod float; +mod g16; +mod g16_compute; +mod g16_drm; +mod g16_fw; +mod g16_platform; +mod g16_power; +mod g16_render; +mod g16_stamp; +mod g16_tvb; +mod g16_vm; +#[path = "m4_mem.rs"] +mod mem; +#[path = "m4_pgtable.rs"] +mod pgtable; +#[path = "m4_util.rs"] +mod util; + +use kernel::io::Io; +use kernel::{c_str, device::Core, devres::Devres, io::mem::IoMem, of, platform, prelude::*}; + +const SGX_SIZE: usize = 0x4000000; +const ID_VERSION: usize = 0xd04000; +const ID_COUNTS_1: usize = 0xd04010; +const ID_COUNTS_2: usize = 0xd04014; +const ID_CLUSTERS: usize = 0xd0401c; + +struct M4Gpu { + _sgx: Option>>>>, + _drm: Option, +} + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [(of::DeviceId::new(c_str!("apple,agx-t8132")), ())] +); + +impl platform::Driver for M4Gpu { + type IdInfo = (); + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + _info: Option<&Self::IdInfo>, + ) -> impl PinInit { + let dev = pdev.as_ref(); + if !(11..=331).contains(module_parameters::tvb_max_blocks.value()) { + return Err(EINVAL); + } + + if *module_parameters::probe_only.value() == 0 { + return Ok(Self { + _sgx: None, + _drm: Some(g16_drm::register(pdev)?), + }); + } + let request = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; + let sgx = KBox::pin_init(request.iomap_sized::(), GFP_KERNEL)?; + { + let regs = sgx.try_access().ok_or(ENODEV)?; + let version = regs.relaxed().read32(ID_VERSION); + if version == 0 || version == u32::MAX { + dev_err!(dev, "Invalid GPU ID {:#010x}\n", version); + return Err(ENODEV); + } + dev_info!( + dev, + "Apple M4 GPU recognized: ID={:#010x} counts={:#010x}/{:#010x} clusters={:#010x}\n", + version, + regs.relaxed().read32(ID_COUNTS_1), + regs.relaxed().read32(ID_COUNTS_2), + regs.relaxed().read32(ID_CLUSTERS), + ); + dev_info!( + dev, + "Probe-only driver; GPU firmware execution is disabled\n" + ); + } + g16::validate_boot_resources(pdev)?; + let platform = g16_platform::Platform::new(dev)?; + let hwdata = platform.hwdata()?; + let region_c = platform.region_c()?; + let bundle = platform.bundle(0)?; + dev_info!(dev, + "M4 source configuration: {} register windows, mask={:#x}, max-frequency={} MHz, max-power={} mW, period={}/{} ms/clocks, objects={}/{}/{} bytes\n", + platform.registers.len(), platform.core_mask, platform.freq_a[10], + platform.maximum_power, platform.period_ms, platform.period_clocks, + hwdata.len(), region_c.len(), bundle.len()); + Ok(Self { + _sgx: Some(sgx), + _drm: None, + }) + } +} + +kernel::module_platform_driver! { + type: M4Gpu, + name: "asahi_m4", + authors: ["Gravity Linux Contributors"], + description: "Apple M4 GPU driver", + license: "GPL v2", + params: { + fw_counter_start: u64 { + default: 0, + description: "Initial submission counters for firmware rollover qualification (normally 0)", + }, + fw_trace: u32 { + default: 0, + description: "Trace mask: 1=firmware KTrace/publications, 2=admission saturation, 4=publication batches, 8=queued dependencies", + }, + tvb_max_blocks: u32 { + default: 331, + description: "Maximum retained TVB blocks per context (11 through 331)", + }, + probe_only: u32 { + default: 0, + description: "Identify the GPU without starting firmware or registering DRM", + }, + }, +} diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index d58a3d284da12477de8aa0e6d1de12078b0628e1..068b2c67d9e8b06e2d7a1466c347fbbb4dcb9d28 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -45,4 +45,30 @@ struct F32U { impl F32 { + /// Firmware wire representation. + pub(crate) const fn to_bits(self) -> u32 { + self.0 + } + + /// Truncate a nonnegative finite value to an unsigned firmware field. + pub(crate) fn to_u32_checked(self) -> Option { + let bits = self.to_bits(); + if bits & 0x8000_0000 != 0 && bits & 0x7fff_ffff != 0 { + return None; + } + let exp = ((bits >> 23) & 255) as i32 - 127; + if exp < 0 { + return Some(0); + } + if exp > 31 { + return None; + } + let frac = (bits & 0x7f_ffff) | 0x80_0000; + Some(if exp >= 23 { + frac << (exp - 23) + } else { + frac >> (23 - exp) + }) + } + /// Convert a raw 32-bit representation into an F32 pub(crate) const fn from_bits(u: u32) -> F32 { diff --git a/drivers/gpu/drm/asahi/g16.rs b/drivers/gpu/drm/asahi/g16.rs new file mode 100644 index 0000000000000000000000000000000000000000..46a7b78cf1f94e2fb393f8584df48b2c1b563af8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16.rs @@ -0,0 +1,959 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from Niklas Sheth's linux-m4-integration prototype. + +//! G16 firmware bootstrap. Its 43-bit address space and initialization graph +//! must not be passed to a G13/G14 firmware manager. + +use kernel::io::Io; +use kernel::{c_str, device::Core, platform, prelude::*}; + +const PAGE_SIZE: u64 = 0x4000; + +/// Validate the memory contract supplied by m1n1 before starting firmware. +pub(crate) fn validate_boot_resources(pdev: &platform::Device) -> Result { + let dev = pdev.as_ref(); + let node = dev.of_node().ok_or(EINVAL)?; + let fwnode = dev.fwnode().ok_or(EINVAL)?; + // This is the firmware build exercised by the Python M4 shim. The + // loader's generic macOS-version lookup currently does not recognize it. + let firmware: KVec = node.get_property(c_str!("apple,firmware-build"))?; + if firmware.as_slice() != b"mBoot-18000.161.10\0" { + dev_err!(dev, "G16: unqualified boot firmware identifier\n"); + return Err(ENOTSUPP); + } + let private_vm: KVec = fwnode + .property_read_array_vec(c_str!("apple,rtkit-private-vm-region"), 2)? + .required_by(dev)?; + + if private_vm.len() != 2 { + return Err(EINVAL); + } + if private_vm[0] != 0xffff_fc00_0000_0000 || private_vm[1] != 0x20_0000_0000 { + dev_err!( + dev, + "G16: unsupported firmware private VM {:?}\n", + private_vm + ); + return Err(EINVAL); + } + + dev_info!(dev, "G16: boot firmware mBoot-18000.161.10\n"); + let mut regions = KVec::<(u64, u64)>::new(); + for (name, min_size) in [ + (c_str!("ttbs"), PAGE_SIZE), + (c_str!("pagetables"), PAGE_SIZE), + (c_str!("l2"), PAGE_SIZE), + (c_str!("handoff"), PAGE_SIZE), + (c_str!("firmware"), PAGE_SIZE), + ] { + let res = node.reserved_mem_region_to_resource_byname(name)?; + let base = res.start() as u64; + let size = res.size() as u64; + let end = base.checked_add(size).ok_or(EINVAL)?; + if base == 0 || size < min_size || (base | size) & (PAGE_SIZE - 1) != 0 { + dev_err!( + dev, + "G16: invalid {} region {:#x}+{:#x}\n", + name, + base, + size + ); + return Err(EINVAL); + } + for &(other_base, other_end) in ®ions { + if base < other_end && other_base < end { + dev_err!(dev, "G16: overlapping boot memory regions\n"); + return Err(EINVAL); + } + } + regions.push((base, end), GFP_KERNEL)?; + dev_info!(dev, "G16: {} {:#x}+{:#x}\n", name, base, size); + } + + dev_info!( + dev, + "G16: boot memory contract validated; firmware initialization pending\n" + ); + Ok(()) +} + +use kernel::{ + devres::Devres, + io::{self, mem::IoMem}, + iosys_map::IoSysMapRef, + soc::apple::rtkit, + sync::Arc, + sync::aref::ARef, + types::ForeignOwnable, +}; + +struct BootData { + dev: ARef, + notifications: Arc, +} + +struct CrashBuffer { + map: io::mem::Mem, + offset: usize, + size: usize, + iova: usize, +} + +impl rtkit::Buffer for CrashBuffer { + fn iova(&self) -> Result { + Ok(self.iova) + } + + fn buf(&mut self) -> Result> { + self.map.as_iosys_map(self.offset, self.size) + } +} + +struct BootOps; + +#[vtable] +impl rtkit::Operations for BootOps { + type Data = Arc; + type Buffer = CrashBuffer; + + fn shmem_map( + data: ::Borrowed<'_>, + iova: usize, + size: usize, + ) -> Result { + let node = data.dev.as_ref().of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(c_str!("firmware"))?; + let offset = iova.checked_sub(res.start() as usize).ok_or(EINVAL)?; + if offset.checked_add(size).ok_or(EINVAL)? > res.size() as usize { + return Err(EINVAL); + } + // SAFETY: The loader reserved this RAM for the already loaded GPU + // firmware. Only its advertised crash-buffer subrange is exposed. + let map = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + dev_info!( + data.dev.as_ref(), + "G16: crash buffer {:#x}+{:#x}\n", + iova, + size + ); + Ok(CrashBuffer { + map, + offset, + size, + iova, + }) + } + + fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + if ep == 0x20 && msg == 0x0042_0000_0000_0000 { + // RTKit delivers this callback in process context after the + // mailbox receive barrier. The worker drains the firmware rings. + data.notifications.notify(false); + return; + } + dev_info!( + data.dev.as_ref(), + "G16: firmware message {:#x}:{:#018x}\n", + ep, + msg + ); + } + + fn crashed(data: ::Borrowed<'_>, _log: Option<&[u8]>) { + data.notifications.notify(true); + dev_err!(data.dev.as_ref(), "G16: firmware crashed\n"); + } +} + +/// Resources retained while the autonomous firmware session is running. +/// No render node is published until the submission backend is available. +#[path = "g16_runtime.rs"] +mod render_runtime; +pub(crate) use render_runtime::{FirmwareQueues, Stamp, Support}; + +#[path = "g16_notify.rs"] +mod notify; +pub(crate) use notify::Notifications; + +// Keep the opening barrier transport separate from render queues. Firmware +// retains queue and item references after consuming this bootstrap prefix. +const BOOT_QUEUE: u64 = 0xffff_fc20_c000_4980; +const BOOT_POINTERS: u64 = 0xffff_fc20_0001_50c0; +const BOOT_RING: u64 = 0xffff_fc20_c001_50c0; +const BOOT_JOBS: u64 = 0xffff_fc20_0000_0030; +const BOOT_CONTEXT: u64 = 0xffff_fc20_c050_8080; +const BOOT_BARRIER: u64 = 0xffff_fc20_c052_0080; + +pub(crate) struct Bootstrap { + dev: ARef, + pub(crate) notifications: Arc, + _rtkit: Pin>>, + _asc: Pin>>>, + _sgx: Pin>>>, + _handoff: io::mem::Mem, + _address_space: Option, + _firmware_space: Option, + _opening_client: Option, + timestamp_ranges: KVec>, + opening_notification_pending: bool, + pub(crate) gpu_revision: u32, + pub(crate) core_mask: u32, + pub(crate) maximum_frequency_khz: u32, + tvb_pools: KVec, + render_publication: u64, + next_work_va: u64, + work_arenas: [core::ops::Range; 2], + free_work: [KVec; 2], + compute_publication: u64, + queue_events: Arc, + render_failed: bool, + flights: KVec, + publications: KVec, + batch_works: usize, + context_roots: [Option; 64], + context_users: [u32; 64], + pressure_reported: core::sync::atomic::AtomicU64, + render_support: kernel::sync::Arc, +} + +// SAFETY: The session owns its mapped memory and page tables; none are +// thread-local. Runtime access is exclusive (under the DRM frontend mutex). +// RTKit callbacks access only their independently synchronized callback data. +unsafe impl Send for Bootstrap {} + +impl Drop for Bootstrap { + fn drop(&mut self) { + if let Err(e) = self._rtkit.as_mut().shutdown() { + dev_err!( + self.dev.as_ref(), + "G16: shutdown failed {:?}; retaining GPU mappings\n", + e + ); + // A failed shutdown is not proof that firmware stopped accessing + // these pages. Keep their tables and backing memory alive. + core::mem::forget(self._address_space.take()); + core::mem::forget(self._firmware_space.take()); + core::mem::forget(self._opening_client.take()); + } + } +} + +impl Bootstrap { + pub(crate) fn new(pdev: &platform::Device) -> Result { + validate_boot_resources(pdev)?; + let platform = crate::g16_platform::Platform::new(pdev.as_ref())?; + // All board inputs are checked before any firmware is started. + let mut bundle = platform.bundle(unsafe { kernel::bindings::get_random_u32() })?; + // The firmware Timestamp handler accepts user destinations only in + // this 64 MiB aperture, based at header+0x28. + bundle[0x28..0x30].copy_from_slice(&crate::g16_fw::TIMESTAMP_BASE.to_le_bytes()); + let region_c = platform.region_c()?; + let dev = pdev.as_ref(); + let mut address_space = crate::g16_vm::AddressSpace::new()?; + address_space.check_geometry(dev)?; + address_space.check_borrowed_root(dev)?; + let asc = KBox::pin_init( + pdev.io_request_by_name(c_str!("asc")) + .ok_or(EINVAL)? + .iomap_sized::<0x4000>(), + GFP_KERNEL, + )?; + let sgx = KBox::pin_init( + pdev.io_request_by_name(c_str!("sgx")) + .ok_or(EINVAL)? + .iomap_sized::<0x4000000>(), + GFP_KERNEL, + )?; + let res = dev + .of_node() + .ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("handoff"))?; + // SAFETY: This is the reserved firmware handoff page from the loader, + // not an allocator-owned page. No new DMA address is introduced here. + let handoff = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + let asc_guard = asc.try_access().ok_or(ENODEV)?; + let asc_io = &*asc_guard; + let control = asc_io.relaxed().read32(0x44); + let status = asc_io.relaxed().read32(0x48); + drop(asc_guard); + dev_info!( + dev, + "G16: cold ASC control={:#x} status={:#x}\n", + control, + status + ); + if control & 0x10 != 0 || status & 3 != 2 { + return Err(EBUSY); + } + let sgx_guard = sgx.try_access().ok_or(ENODEV)?; + let sgx_io = &*sgx_guard; + let gpu_id = sgx_io.relaxed().read32(crate::ID_VERSION); + if gpu_id == 0 || gpu_id == u32::MAX { + dev_err!(dev, "Invalid GPU ID {:#010x}\n", gpu_id); + return Err(ENODEV); + } + // ID_VERSION retains the revision byte used by the Asahi UAPI: + // 0x00 = A0, 0x10 = B0, 0x11 = B1. This is not the firmware enum. + let gpu_revision = (gpu_id >> 8) & 0xff; + dev_info!(dev, "G16: GPU ID={:#010x} revision={:#04x}\n", gpu_id, gpu_revision); + for offset in [0x1000104, 0x1000108] { + sgx_io.relaxed().write32(sgx_io.relaxed().read32(offset) | 1, offset); + } + sgx_io.relaxed().write32(6, 0xd06030); + drop(sgx_guard); + // SAFETY: The region validation guarantees a full aligned page. The + // coprocessor is stopped, so these host-owned bootstrap fields are + // not concurrently accessed by firmware. + unsafe { + handoff + .ptr() + .cast::() + .write_volatile(0x4b1d000000000002); + handoff + .ptr() + .add(0x18) + .cast::() + .write_volatile(u32::MAX); + handoff.ptr().add(0x640).cast::().write_volatile(0); + for off in (0..PAGE_SIZE as usize).step_by(64) { + core::arch::asm!("dc cvac, {addr}", addr = in(reg) handoff.ptr().add(off)); + } + } + crate::mem::sync(); + + let notifications = Arc::pin_init(Notifications::new(), GFP_KERNEL)?; + let data = Arc::new( + BootData { + dev: pdev.into(), + notifications: notifications.clone(), + }, + GFP_KERNEL, + )?; + let mut rtkit = KBox::pin( + rtkit::RtKit::::new(dev, None, 0, data)?, + GFP_KERNEL, + )?; + rtkit.as_mut().set_early_crashlog(); + let asc_guard = asc.try_access().ok_or(ENODEV)?; + asc_guard.relaxed().write32(control | 0x10, 0x44); + drop(asc_guard); + crate::mem::sync(); + rtkit.as_mut().boot()?; + for ep in [0x20, 0x21] { + if !rtkit.as_mut().has_endpoint(ep) { + return Err(ENODEV); + } + rtkit.as_mut().start_endpoint(ep)?; + } + dev_info!( + dev, + "G16: autonomous RTKit boot complete; firmware endpoints ready\n" + ); + let mut firmware_space = crate::g16_vm::FirmwareSpace::attach(dev)?; + firmware_space.alloc( + crate::g16_fw::ROOT_ADDRESS, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RO, + )?; + firmware_space.write( + crate::g16_fw::ROOT_ADDRESS, + &crate::g16_fw::RootPointers::bootstrap().encode(), + )?; + firmware_space.alloc( + crate::g16_fw::BUNDLE_ADDRESS, + crate::g16_fw::BUNDLE_SIZE + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_PRIV_RW, + )?; + firmware_space.write(crate::g16_fw::BUNDLE_ADDRESS, &bundle)?; + // All root support objects and receive rings occupy this shared extent. + firmware_space.alloc( + 0xffff_fc20_0003_0000, + 0xc4000, + crate::pgtable::prot::PROT_FW_SHARED_RW, + )?; + let pointers = crate::g16_fw::RootPointers::bootstrap(); + for va in [pointers.region_a, pointers.region_c] { + firmware_space.alloc( + va, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RO, + )?; + } + for va in [pointers.status_a, pointers.status_b] { + firmware_space.write(va + 4, &1u32.to_le_bytes())?; + } + firmware_space.write(pointers.region_c, ®ion_c)?; + // Extra policy objects, computed descriptors, PB/UMA tables and + // scheduler counters begin zero, exactly as the current shim. + firmware_space.alloc( + 0xffff_fc20_c050_0000, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_PRIV_RW, + )?; + for r in &platform.registers { + let prot = if r.flags & 2 != 0 { + crate::pgtable::prot::PROT_FW_MMIO_RW + } else { + crate::pgtable::prot::PROT_FW_MMIO_RO + }; + if r.slot == 3 { + for (i, physical) in platform.mcc.iter().enumerate() { + firmware_space.map_external( + r.address + i as u64 * u64::from(r.stride), + *physical, + r.stride as usize, + prot, + )?; + } + } else { + let offset = r.physical & (PAGE_SIZE - 1); + let size = (offset + u64::from(r.size) + PAGE_SIZE - 1) & !(PAGE_SIZE - 1); + firmware_space.map_external( + r.address - offset, + r.physical - offset, + size as usize, + prot, + )?; + } + } + // Work-channel cursor pages use shared attributes, unlike other MMIO. + firmware_space.map_external( + 0xffff_fc20_0002_0000, + 0x300d60000, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RO, + )?; + firmware_space.map_external( + 0xffff_fc20_0002_8000, + 0x300d64000, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RW, + )?; + for (va, size) in [ + (0xffff_fc20_c002_0000, 0x8c000), + (0xffff_fc20_c00b_0000, 0xc0000), + (0xffff_fc20_c017_8000, 0x94000), + (0xffff_fc20_c021_0000, 0x84000), + (0xffff_fc20_c029_8000, 0xac000), + (0xffff_fc20_c034_8000, 0x4000), + (0xffff_fc20_c035_0000, 0x4000), + (0xffff_fc20_c035_8000, 0x8000), + (0xffff_fc20_c036_8000, 0x8000), + (0xffff_fc20_c037_8000, 0xc000), + (0xffff_fc20_c038_8000, 0x34000), + (0xffff_fc20_c03c_0000, 0xc4000), + (0xffff_fc20_c048_8000, 0x34000), + (0xffff_fc20_c04c_0000, 0x10000), + (0xffff_fc20_c051_0000, 0x4000), + (0xffff_fc20_c053_0000, 0x4000), + (0xffff_fc20_c05e_8000, 0x4000), + (0xffff_fc20_c05e_c000, 0x4000), + ] { + firmware_space.alloc(va, size, crate::pgtable::prot::PROT_FW_PRIV_RW)?; + } + for va in [ + 0xffff_fc20_0011_0000, + 0xffff_fc20_0011_8000, + 0xffff_fc20_0012_0000, + 0xffff_fc20_0012_8000, + 0xffff_fc20_0013_0000, + 0xffff_fc20_0015_0000, + 0xffff_fc20_0014_0000, + 0xffff_fc20_001b_0000, + 0xffff_fc20_001c_0000, + 0xffff_fc20_001c_4000, + 0xffff_fc20_001c_8000, + 0xffff_fc20_001c_c000, + 0xffff_fc20_c052_4000, + ] { + firmware_space.alloc( + va, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RW, + )?; + } + firmware_space.alloc( + 0xffff_fc20_001b_8000, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RO, + )?; + // Shared support version and read-only resume records also serve the + // render support object; construct them before the firmware starts. + firmware_space.write(0xffff_fc20_001b_0000, &1u32.to_le_bytes())?; + for record in 0..96u64 { + firmware_space.write( + 0xffff_fc20_001b_8000 + record * 0x40, + &0x0000_0070_0000_1822u64.to_le_bytes(), + )?; + } + for (va, size) in [ + (0x70_0000_0000, 0x8c000), + (0x70_0009_0000, 0xc0000), + (0x70_0015_8000, 0x94000), + (0x70_001f_0000, 0x84000), + ] { + address_space.alloc_low(va, size, crate::pgtable::prot::PROT_GPU_SHARED_RO)?; + } + for (low, high) in [ + (0x70_0027_8000, 0xffff_fc20_0012_8000), + (0x70_0028_0000, 0xffff_fc20_0013_0000), + ] { + let pa = firmware_space.physical(high)?; + address_space.low.map_pages( + low..low + PAGE_SIZE, + pa, + crate::pgtable::prot::PROT_GPU_SHARED_RW, + false, + )?; + if address_space.low.translate(low)? != Some(pa) { + return Err(EIO); + } + } + firmware_space.alloc_render_support()?; + use crate::g16_compute as compute; + firmware_space.alloc( + compute::QUEUE, + 0xc000, + crate::pgtable::prot::PROT_FW_PRIV_RW, + )?; + firmware_space.alloc( + compute::SHARED, + 0x4000, + crate::pgtable::prot::PROT_FW_SHARED_RW, + )?; + let render = crate::g16_render::Addresses::bootstrap(); + // The Work register windows are firmware-owned pages exposed through + // context zero's low root. Replace the initial zero backing explicitly. + for (alias, work) in [ + (render.tiling_register_alias, render.tiling_work), + (render.fragment_register_alias, render.fragment_work), + ] { + let base = alias & !(PAGE_SIZE - 1); + let pa = firmware_space.physical(work & !(PAGE_SIZE - 1))?; + address_space.low.unmap_pages(base..base + PAGE_SIZE)?; + address_space.low.map_pages( + base..base + PAGE_SIZE, + pa, + crate::pgtable::prot::PROT_GPU_SHARED_RW, + false, + )?; + } + address_space.sync(); + // Reserve the transport before firmware can cache a missing mapping. + for (va, size, access) in [ + ( + 0xffff_fc20_c000_0000, + 0x8000, + crate::pgtable::prot::PROT_FW_PRIV_RW, + ), + ( + 0xffff_fc20_0001_0000, + 0x8000, + crate::pgtable::prot::PROT_FW_SHARED_RW, + ), + ( + 0xffff_fc20_c001_0000, + 0x8000, + crate::pgtable::prot::PROT_FW_PRIV_RW, + ), + ( + 0xffff_fc20_0000_0000, + 0x4000, + crate::pgtable::prot::PROT_FW_SHARED_RW, + ), + ( + 0xffff_fc20_c050_8000, + 0x4000, + crate::pgtable::prot::PROT_FW_SHARED_RW, + ), + ( + 0xffff_fc20_c052_0000, + 0x4000, + crate::pgtable::prot::PROT_FW_PRIV_RW, + ), + ( + 0xffff_fc20_0013_8000, + 0x4000, + crate::pgtable::prot::PROT_FW_SHARED_RW, + ), + ] { + firmware_space.alloc(va, size, access)?; + } + use crate::g16_fw::queue as q; + firmware_space.write(BOOT_POINTERS, &q::pointers(0x500, 2))?; + firmware_space.write(BOOT_JOBS, &q::jobs(BOOT_JOBS))?; + firmware_space.write(BOOT_CONTEXT, &q::context())?; + firmware_space.write( + BOOT_QUEUE, + &q::Queue { + pointers: BOOT_POINTERS, + ring: BOOT_RING, + jobs: BOOT_JOBS, + private: BOOT_QUEUE + 0xb0, + context: BOOT_CONTEXT, + uuid: 1, + } + .encode(), + )?; + firmware_space.write(BOOT_BARRIER, &q::barrier(q::STAMP, 0, 0, 1, 1))?; + for off in [0, 8] { + firmware_space.write(BOOT_RING + off, &BOOT_BARRIER.to_le_bytes())?; + } + firmware_space.sync(); + let mut opening_client = crate::g16_vm::AddressSpace::new()?; + opening_client.alloc_low( + 0x70_0000_0000, + 0x400000, + crate::pgtable::prot::PROT_GPU_SHARED_RW, + )?; + opening_client.alloc_low( + crate::g16_fw::OPERAND_TABLE, + 0x10000, + crate::pgtable::prot::PROT_GPU_SHARED_RW, + )?; + for index in 0..20 { + opening_client.alloc_low( + crate::g16_fw::operand_page(index * 512), + 0x200000, + crate::pgtable::prot::PROT_GPU_SHARED_RW, + )?; + } + for first in (0..20 * 512).step_by(32) { + let mut row = [0u8; 256]; + for i in 0..32 { + row[i * 8..i * 8 + 8] + .copy_from_slice(&crate::g16_fw::operand_page(first + i).to_le_bytes()); + } + opening_client.write_low(0x70_0000_0000 + (first * 8) as u64, &row)?; + } + for (base, size) in compute::private_ranges() { + opening_client.alloc_low(base, size, crate::pgtable::prot::PROT_GPU_SHARED_RW)?; + } + for buffer in 0..compute::BUFFER_COUNT { + for first in (0..compute::BUFFER_SIZE / 0x1000).step_by(32) { + let mut row = [0; 256]; + for i in 0..32 { + let va = compute::BUFFER_BASE + + buffer as u64 * compute::BUFFER_STRIDE + + (first + i) as u64 * 0x1000; + row[i * 8..i * 8 + 8].copy_from_slice(&va.to_le_bytes()); + } + opening_client.write_low( + compute::DIRECTORY + + (buffer * (compute::BUFFER_SIZE / 0x1000) + first) as u64 * 8, + &row, + )?; + } + } + firmware_space.write_live(compute::SUPPORT, &compute::support())?; + opening_client.sync(); + dev_info!( + dev, + "G16: Linux-built root/main descriptors mapped at {:#x}; not yet published\n", + crate::g16_fw::ROOT_ADDRESS + ); + let mut session = Self { + dev: pdev.into(), + notifications, + _rtkit: rtkit, + _asc: asc, + _sgx: sgx, + _handoff: handoff, + _address_space: Some(address_space), + _firmware_space: Some(firmware_space), + _opening_client: Some(opening_client), + timestamp_ranges: KVec::new(), + opening_notification_pending: true, + gpu_revision, + core_mask: platform.core_mask, + maximum_frequency_khz: platform.freq_a[10] * 1000, + tvb_pools: KVec::new(), + render_publication: *crate::module_parameters::fw_counter_start.value(), + next_work_va: 0xffff_fc22_0000_0000, + work_arenas: [0..0, 0..0], + free_work: [KVec::new(), KVec::new()], + compute_publication: *crate::module_parameters::fw_counter_start.value(), + queue_events: Arc::new(core::sync::atomic::AtomicU64::new(0), GFP_KERNEL)?, + render_failed: false, + flights: KVec::new(), + publications: KVec::new(), + batch_works: 0, + context_roots: [None; 64], + context_users: [0; 64], + pressure_reported: core::sync::atomic::AtomicU64::new(0), + render_support: kernel::sync::Arc::new( + core::sync::atomic::AtomicU64::new(0), + GFP_KERNEL, + )?, + }; + if *crate::module_parameters::fw_trace.value() & 1 != 0 { + // Region C starts with the firmware KTrace mask, as in the shim. + session + ._firmware_space + .as_mut() + .ok_or(EIO)? + .write_live(0xffff_fc20_000f_8000, &u32::MAX.to_le_bytes())?; + } + session.publish(pdev)?; + session.open_control(pdev)?; + session.submit_barrier(pdev)?; + Ok(session) + } + + fn submit_barrier(&mut self, pdev: &platform::Device) -> Result { + use crate::g16_fw::queue as q; + let dev = pdev.as_ref(); + let firmware = self._firmware_space.as_mut().ok_or(EINVAL)?; + let sgx_guard = self._sgx.try_access().ok_or(ENODEV)?; + let sgx = &*sgx_guard; + for addr in [0xffff_fc20_0003_f190, 0xffff_fc20_0003_f198] { + firmware.write_live(addr, &0u32.to_le_bytes())?; + } + let before = ( + sgx.relaxed().read32(0xd64040), + sgx.relaxed().read32(0xd64048), + sgx.relaxed().read32(0xd60020), + ); + if before != (0, 0, 0) { + return Err(EBUSY); + } + // Pair-zero fragment channel, with two satisfied barrier items. + firmware.write_live( + crate::g16_fw::BUNDLE_ADDRESS + 0x16240, + &q::channel(BOOT_QUEUE, 2, 1, 1, true, 1), + )?; + sgx.relaxed().write32(1, 0xd60020); + crate::mem::sync(); + drop(sgx_guard); + let mut committed = false; + for _ in 0..3000 { + let cursor = firmware.read_u32(crate::g16_fw::CONTROL_DATA + 0x48)?; + let aux = firmware.read_u32(crate::g16_fw::CONTROL_AUX)?; + if cursor == 20 * 8 && (aux == 0 || aux == 1) { + committed = true; + break; + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + if !committed { + dev_err!( + dev, + "G16: first-work registration stalled at {:#x}/{:#x}\n", + firmware.read_u32(crate::g16_fw::CONTROL_DATA + 0x48)?, + firmware.read_u32(crate::g16_fw::CONTROL_AUX)? + ); + return Err(ETIMEDOUT); + } + self._rtkit + .as_mut() + .send_message(0x21, 0x0083_0000_0000_0001)?; + for _ in 0..3000 { + let sgx_guard = self._sgx.try_access().ok_or(ENODEV)?; + let sgx = &*sgx_guard; + if sgx.relaxed().read32(0xd64040) == 1 + && sgx.relaxed().read32(0xd64048) == 1 + && firmware.read_u32(BOOT_POINTERS + 0x30)? == 2 + { + // Barriers consume the inner ring without advancing its + // separate completion cursor, as in the working source path. + dev_info!(dev, "G16: barrier channel retired; two items consumed\n"); + return Ok(()); + } + drop(sgx_guard); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + let sgx_guard = self._sgx.try_access().ok_or(ENODEV)?; + let sgx = &*sgx_guard; + dev_err!( + dev, + "G16: barrier timeout; channel {}/{}/{} item done/read {}/{}\n", + sgx.relaxed().read32(0xd64040), + sgx.relaxed().read32(0xd64048), + sgx.relaxed().read32(0xd60020), + firmware.read_u32(BOOT_POINTERS)?, + firmware.read_u32(BOOT_POINTERS + 0x30)? + ); + Err(ETIMEDOUT) + } + + fn open_control(&mut self, pdev: &platform::Device) -> Result { + let dev = pdev.as_ref(); + let firmware = self._firmware_space.as_mut().ok_or(EINVAL)?; + let client = self._opening_client.as_ref().ok_or(EINVAL)?; + let res = dev + .of_node() + .ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("ttbs"))?; + // SAFETY: This is the loader's validated context table reservation. + let ttbs = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + unsafe { + // Context one has ASID one, with independently allocated roots. + ttbs.ptr() + .add(16) + .cast::() + .write_volatile(client.low.ttb() | (1 << 48) | 1); + ttbs.ptr() + .add(24) + .cast::() + .write_volatile(client.high.ttb() | (1 << 48) | 1); + core::arch::asm!("dc cvac, {addr}", addr = in(reg) ttbs.ptr()); + } + crate::mem::sync(); + firmware.alloc( + crate::g16_fw::CONTROL_DATA, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_PRIV_RW, + )?; + firmware.alloc( + crate::g16_fw::CONTROL_AUX, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RW, + )?; + firmware.write(crate::g16_fw::CONTROL_DATA, &crate::g16_fw::opening_flist())?; + // Own distinct shared pages for the mapping-notification state/ring. + firmware.alloc( + 0xffff_fc21_8100_0000, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RW, + )?; + firmware.alloc( + 0xffff_fc21_8100_8000, + PAGE_SIZE as usize, + crate::pgtable::prot::PROT_FW_SHARED_RW, + )?; + firmware.sync(); + firmware.write_live( + crate::g16_fw::BUNDLE_ADDRESS + 0x1df40, + &crate::g16_fw::opening_control(), + )?; + let sgx_guard = self._sgx.try_access().ok_or(ENODEV)?; + let sgx = &*sgx_guard; + let before = ( + sgx.relaxed().read32(0xd640c0), + sgx.relaxed().read32(0xd640c8), + sgx.relaxed().read32(0xd60060), + ); + if before != (1, 1, 1) { + dev_err!(dev, "G16: unexpected opening cursors {:?}\n", before); + return Err(EIO); + } + sgx.relaxed().write32(3, 0xd60060); + drop(sgx_guard); + for (address, value) in [ + (0xffff_fc20_0003_4f88, 0xffff_fc21_8100_0000u64), + (0xffff_fc20_0003_4f90, 0xffff_fc21_8100_8000u64), + ] { + firmware.write_live(address, &value.to_le_bytes())?; + } + firmware.write_live(0xffff_fc20_0003_4fd4, &1u32.to_le_bytes())?; + crate::mem::tlbi_asid(1); + crate::mem::sync(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(100)); + self._rtkit + .as_mut() + .send_message(0x21, 0x0084_0000_0000_0011)?; + for _ in 0..3000 { + let sgx_guard = self._sgx.try_access().ok_or(ENODEV)?; + let sgx = &*sgx_guard; + let cursors = ( + sgx.relaxed().read32(0xd640c0), + sgx.relaxed().read32(0xd640c8), + sgx.relaxed().read32(0xd60060), + ); + if cursors == (3, 3, 3) && firmware.read_u32(0xffff_fc20_0003_f19c)? == 1 { + dev_info!( + dev, + "G16: opening control registration retired; cursors {:?}\n", + cursors + ); + return Ok(()); + } + drop(sgx_guard); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + let sgx_guard = self._sgx.try_access().ok_or(ENODEV)?; + let sgx = &*sgx_guard; + dev_err!( + dev, + "G16: control timeout; cursors {}/{}/{} completion {}\n", + sgx.relaxed().read32(0xd640c0), + sgx.relaxed().read32(0xd640c8), + sgx.relaxed().read32(0xd60060), + firmware.read_u32(0xffff_fc20_0003_f19c)? + ); + Err(ETIMEDOUT) + } + + fn publish(&mut self, pdev: &platform::Device) -> Result { + let dev = pdev.as_ref(); + let address_space = self._address_space.as_ref().ok_or(EINVAL)?; + let firmware_space = self._firmware_space.as_ref().ok_or(EINVAL)?; + let res = dev + .of_node() + .ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("ttbs"))?; + // SAFETY: This is the validated reserved hardware context table. + let ttbs = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + let sgx_guard = self._sgx.try_access().ok_or(ENODEV)?; + let sgx = &*sgx_guard; + let main = crate::g16_fw::MainConfig::bootstrap(crate::g16_fw::BUNDLE_ADDRESS); + for channel in &main.channels[..12] { + for va in channel.state { + let offset = if va >= 0xffff_fc20_0002_8000 { + 0xd64000 + (va - 0xffff_fc20_0002_8000) as usize + } else { + 0xd60000 + (va - 0xffff_fc20_0002_0000) as usize + }; + sgx.try_write32(0, offset)?; + if sgx.try_read32(offset)? != 0 { + return Err(EIO); + } + } + } + // Slot zero (opcode 0x16) is already embedded in the main object. + for (offset, value) in [(0xd640c0, 0), (0xd640c8, 0), (0xd60060, 1)] { + sgx.try_write32(value, offset)?; + if sgx.try_read32(offset)? != value { + return Err(EIO); + } + } + drop(sgx_guard); + // Slot zero's high root is empty. Firmware keeps using its separate + // bootstrap root from the loader reservation, not this context entry. + unsafe { + ttbs.ptr() + .cast::() + .write_volatile(address_space.low.ttb() | 1); + ttbs.ptr() + .add(8) + .cast::() + .write_volatile(address_space.high.ttb() | 1); + core::arch::asm!("dc cvac, {addr}", addr = in(reg) ttbs.ptr()); + } + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); + self._rtkit + .as_mut() + .send_message(0x20, 0x0081_0c20_0010_0000)?; + dev_info!(dev, "G16: published Linux-built initialization graph\n"); + for _ in 0..3000 { + if firmware_space.read_u32(crate::g16_fw::RootPointers::bootstrap().status_a + 0x14)? + == 1 + { + dev_info!(dev, "G16: firmware accepted initialization graph\n"); + return Ok(()); + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + dev_err!( + dev, + "G16: initialization timeout; status {:#x}/{:#x}/{:#x}\n", + firmware_space.read_u32(0xffff_fc20_0003_f190)?, + firmware_space.read_u32(0xffff_fc20_0003_f194)?, + firmware_space.read_u32(0xffff_fc20_0003_f198)? + ); + Err(ETIMEDOUT) + } +} diff --git a/drivers/gpu/drm/asahi/g16_compute.rs b/drivers/gpu/drm/asahi/g16_compute.rs new file mode 100644 index 0000000000000000000000000000000000000000..da6ccd06fd90f27278658c9e693b7e6754b53a60 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_compute.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from Niklas Sheth's linux-m4-integration prototype. + +//! T8132 compute work and microsequence encoding. All links are supplied by +//! the owner of the work allocation, independently of Rust structure layout. + +pub(crate) const QUEUE: u64 = 0xffff_fc20_c500_0000; +pub(crate) const STATS: u64 = QUEUE + 0x8000; +pub(crate) const SHARED: u64 = 0xffff_fc20_0200_0000; + +// Device-owned opening namespace. Client roots share only the FList directory +// and spill pool; each command receives distinct scratch/marker VAs instead. +pub(crate) const DIRECTORY: u64 = 0x70_02cc_0000; +pub(crate) const DIRECTORY_SIZE: usize = 0x18000; +pub(crate) const BUFFER_BASE: u64 = 0x70_0310_8000; +pub(crate) const BUFFER_STRIDE: u64 = 0x208000; +pub(crate) const BUFFER_SIZE: usize = 0x200000; +pub(crate) const BUFFER_COUNT: usize = 16; +pub(crate) const SCRATCH: u64 = 0x70_030e_0000; +pub(crate) const MARKER: u64 = 0x10_00c6_0000; +pub(crate) const SUPPORT: u64 = 0xffff_fc20_c05e_8000; +pub(crate) const SHARED_STATE: u64 = 0xffff_fc20_001b_0000; +pub(crate) const ENTRY_OFFSET: u64 = 0x1a00; + +/// Drain/invalidate compute caches before entering the caller's CDM stream. +/// Back-to-back M4 Works require the full barrier used by Mesa's CDM helper; +/// 0x60000168 does not preserve dependent SSBO writes between Works. +pub(crate) fn entry(stream: u64) -> [u8; 12] { + let mut out = [0; 12]; + u32_at(&mut out, 0, 0x600fffff); + u32_at(&mut out, 4, 0x20000000 | (stream >> 32) as u32); + u32_at(&mut out, 8, stream as u32); + out +} + +pub(crate) fn private_ranges() -> [(u64, usize); 19] { + core::array::from_fn(|i| match i { + 0 => (DIRECTORY, DIRECTORY_SIZE), + 1 => (SCRATCH, 0x20000), + 2 => (MARKER, 0x4000), + _ => (BUFFER_BASE + (i - 3) as u64 * BUFFER_STRIDE, BUFFER_SIZE), + }) +} + +pub(crate) fn support() -> [u8; 0x100] { + let mut out = [0; 0x100]; + let pages = (BUFFER_COUNT * BUFFER_SIZE / 0x1000) as u32; + u64_at(&mut out, 0, 2); + // +8 selects the hardware FList slot; +0 is a separate pool identity. + // Sharing slot zero with render corrupts concurrently active pool state. + u32_at(&mut out, 8, 2); + u64_at(&mut out, 0x14, DIRECTORY); + u32_at(&mut out, 0x1c, (DIRECTORY_SIZE / 8) as u32); + u32_at(&mut out, 0x24, pages % (DIRECTORY_SIZE / 8) as u32); + u32_at(&mut out, 0x2c, pages); + u64_at(&mut out, 0x4c, SHARED_STATE); + out +} + +fn u32_at(out: &mut [u8], off: usize, value: u32) { + out[off..off + 4].copy_from_slice(&value.to_le_bytes()); +} +fn u64_at(out: &mut [u8], off: usize, value: u64) { + out[off..off + 8].copy_from_slice(&value.to_le_bytes()); +} + +#[derive(Clone, Copy)] +pub(crate) struct Parameters { + pub(crate) cdm: u64, + pub(crate) cdm_end: u64, + pub(crate) sampler: u64, + pub(crate) sampler_count: u32, + pub(crate) scratch: u64, + pub(crate) marker: u64, + pub(crate) save_area: u64, +} + +#[derive(Clone, Copy)] +pub(crate) struct Addresses { + pub(crate) work: u64, + pub(crate) microsequence: u64, + pub(crate) register_table: u64, + pub(crate) queue: u64, + pub(crate) stats: u64, + pub(crate) support: u64, + pub(crate) notifier: u64, + pub(crate) threshold: u64, + pub(crate) driver_stamp: u64, + pub(crate) firmware_stamp: u64, + pub(crate) timestamp_start: u64, + pub(crate) timestamp_end: u64, + pub(crate) context: u32, + pub(crate) identity: u64, + pub(crate) register_identity: u64, + pub(crate) counter: u64, + pub(crate) event_generation: u32, + pub(crate) stamp: u32, + pub(crate) event: u32, + pub(crate) ordinal: u32, + pub(crate) cdm_entry: Option, +} + +impl Addresses { + /// Work storage is retained by its owner. Firmware-width fields wrap + /// independently of the full software publication and queue counters. + pub(crate) fn publication(index: u64, ordinal: u64, work: u64, alias: u64) -> Self { + let count = index.wrapping_add(1); + Self { + work, + microsequence: work + 0x1000, + register_table: alias + 0x20, + queue: QUEUE, + stats: STATS, + support: SUPPORT, + notifier: work + 0x1400, + threshold: work + 0x1800, + driver_stamp: work + 0x1600, + firmware_stamp: work + 0x1680, + timestamp_start: work + 0x1700, + timestamp_end: work + 0x1708, + context: 1, + identity: (1u64 << 32) | u64::from(count as u32), + register_identity: (1u64 << 32) + 1, + counter: count, + event_generation: count as u32, + stamp: crate::g16_stamp::value(ordinal, 0x100), + event: 4, + ordinal: ordinal as u32, + cdm_entry: None, + } + } + + pub(crate) fn notifier(&self, count: u32) -> [u8; 0x100] { + let mut out = [0; 0x100]; + u64_at(&mut out, 0, self.threshold); + for (off, val) in [ + (8, self.event_generation), + (12, count), + (16, 0x50), + (0x24, self.context), + ] { + u32_at(&mut out, off, val); + } + out + } +} + +impl Parameters { + fn valid(&self) -> bool { + self.cdm != 0 + && self.cdm & 3 == 0 + && self.cdm_end > self.cdm + && self.cdm_end & 3 == 0 + && self.sampler & 7 == 0 + && self.sampler_count < u32::MAX + && (self.sampler_count == 0) == (self.sampler == 0) + } + + pub(crate) fn work(&self, a: &Addresses) -> Option<[u8; 0x8c0]> { + if !self.valid() { + return None; + } + let mut out = [0; 0x8c0]; + u32_at(&mut out, 0, 3); + u64_at(&mut out, 4, a.counter); + u32_at(&mut out, 12, a.context); + u64_at(&mut out, 16, a.notifier); + let registers = [ + (0x1a510, self.save_area), + (0x1a420, a.cdm_entry.unwrap_or(self.cdm)), + (0x1a4d0, self.save_area + 0x1480), + (0x1a4d8, self.save_area + 0x1488), + (0x1a4e0, self.save_area + 0x1490), + (0x1a4e8, self.save_area + 0x1498), + (0x1a440, 0x154024201), + (0x1a458, 0x10c08ae0), + (0x101d9, 0x1c), + (0x1a089, 0), + (0x1a091, 0), + (0x1a059, 0), + (0x1a061, 0), + (0x1a0b9, 0), + (0x1a0c1, 0), + (0x101d1, 0), + (0x0d479, 0), + (0x1a0e9, 8), + (0x107a1, 0x00ff0000), + (0x0a599, 0x20000400020), + (0x0d411, 0x200000001), + (0x1a540, a.register_identity), + (0x014a9, a.register_identity), + (0x0a351, a.register_identity), + ]; + for (index, (reg, value)) in registers.iter().enumerate() { + u32_at(&mut out, 0x20 + index * 12, *reg); + u64_at(&mut out, 0x24 + index * 12, *value); + } + for (off, value) in [ + (0x720, a.register_table), + (0x760, a.microsequence), + (0x798, self.save_area), + (0x7a0, self.cdm_end - 4), + ( + 0x7e8, + if self.sampler_count == 0 { + 0xffff_ffff + } else { + self.sampler + }, + ), + (0x800, a.driver_stamp), + (0x808, a.firmware_stamp), + (0x834, a.timestamp_start), + (0x83c, a.timestamp_end), + (0x86a, a.support), + (0x883, 0), + ] { + u64_at(&mut out, off, value); + } + for (off, value) in [ + (0x728, 24 | (288 << 16)), + (0x768, 0x300), + (0x7c8, 0x54024201), + (0x7cc, 1), + (0x7e0, (a.identity >> 32) as u32), + (0x7f0, self.sampler_count), + ( + 0x7f4, + if self.sampler_count == 0 { + 0 + } else { + self.sampler_count + 1 + }, + ), + (0x810, a.stamp), + (0x81c, 1), // Flush notifier stamps before storage reuse. + (0x814, a.event), + (0x820, a.identity as u32), + (0x828, a.ordinal), + (0x874, 0x21b00), + (0x87c, 0x11700), + ] { + u32_at(&mut out, off, value); + } + out[0x872] = 1; + out[0x88b] = 1; + Some(out) + } + + pub(crate) fn microsequence(&self, a: &Addresses) -> [u8; 0x300] { + let mut out = [0; 0x300]; + for (off, value) in [ + (0, 0x0b), + (0x2c, a.context), + (0x30, 1), + (0x34, a.event_generation), + (0x50, a.identity as u32), + (0x1b0, a.counter as u32), + (0x1b8, a.event), + (0x208, 1), + (0x258, 0x0c), + (0x26c, a.context), + (0x27c, a.identity as u32), + (0x28c, a.stamp), + (0x2bc, (-0x258i32) as u32), + (0x2d4, 0x40000002), + ] { + u32_at(&mut out, off, value); + } + for (off, value) in [ + (0x14, a.work + 0x20), + (0x1c, a.stats), + (0x24, a.queue), + (0x44, a.work + 0x76c), + (0x160, a.support), + (0x178, self.marker), + (0x188, self.scratch), + (0x190, self.scratch + 0xf400), + (0x198, self.scratch + 0x1e800), + (0x1a0, self.scratch + 0x1f000), + (0x1a8, a.work + 0x8ac), + (0x25c, a.stats), + (0x264, a.queue), + (0x270, a.work + 0x76c), + (0x284, a.firmware_stamp), + (0x2b4, a.microsequence + 0x160), + (0x2c1, a.work + 0x8ac), + (0x2c9, a.work + 0x88c), + ] { + u64_at(&mut out, off, value); + } + for (base, opcode, update) in [(0x1bc, 0x80000003, 0x834), (0x20c, 3, 0x83c)] { + u32_at(&mut out, base, opcode); + for (off, value) in [ + (4, a.work + 0x82c), + (12, a.work + 0x834), + (20, a.work + update), + (28, a.queue), + (44, a.work + 0x89c), + (52, a.work + 0x7dc), + ] { + u64_at(&mut out, base + off, value); + } + u32_at(&mut out, base + 68, a.identity as u32); + } + out + } +} diff --git a/drivers/gpu/drm/asahi/g16_drm.rs b/drivers/gpu/drm/asahi/g16_drm.rs new file mode 100644 index 0000000000000000000000000000000000000000..70d3a643765d13408bb813bba9fa6cbdd4947c14 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_drm.rs @@ -0,0 +1,1803 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from Niklas Sheth's linux-m4-integration prototype. + +//! G16 DRM frontend. Firmware initialization and queue ownership stay in the +//! G16 runtime; GEM storage uses the common DRM shmem implementation. + +use crate::{g16_compute as compute, g16_render as render}; +use core::{ + ops::Range, + sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, Ordering}, +}; +use kernel::dma_fence::{self, RawDmaFence}; +use kernel::drm::sched; +use kernel::drm::{ + gem::{shmem, BaseObject, DriverObject}, + ioctl, +}; +use kernel::sync::{Arc, Mutex}; +use kernel::sync::aref::ARef; +use kernel::uaccess::{UserPtr, UserSlice}; +use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; +use kernel::{bindings, c_str, device::Core, drm, platform, prelude::*, uapi}; + +enum Command { + Render(uapi::drm_asahi_cmd_render), + Compute(uapi::drm_asahi_cmd_compute), +} +#[derive(Clone, Copy)] +enum Parameters { + Render(render::Parameters), + Compute(compute::Parameters), +} + +pub(crate) struct Driver; +pub(crate) type Device = drm::Device; +pub(crate) type DeviceRef = ARef; +type DrmFile = drm::File; +type Object = shmem::Object; +const VM_END: u64 = (1 << 42) - 0x8000; +static NEXT_FILE: AtomicU64 = AtomicU64::new(1); +static NEXT_EXECUTION: AtomicU64 = AtomicU64::new(1); + +#[derive(Clone)] +struct Mapping { + range: Range, + permissions: u32, + offset: u64, + single: bool, + sg: Arc>, +} + +impl Mapping { + fn cover(mappings: &[Self], start: u64, size: u64, permissions: u32) -> Result { + let end = start.checked_add(size).ok_or(EINVAL)?; + if size == 0 { + return Err(EINVAL); + } + let mut cursor = start; + while cursor < end { + let mapping = mappings + .iter() + .find(|m| m.range.contains(&cursor) && m.permissions & permissions == permissions) + .ok_or(EINVAL)?; + cursor = end.min(mapping.range.end); + } + Ok(()) + } +} + +struct Vm { + id: u32, + visible: bool, + kernel: Range, + // Tables must be removed before the references keeping GEM pages pinned. + space: Arc>, + compute_space: Arc>, + mappings: KVec, + pending: KVec, + reservation: ARef, + render_support: Option>, + render_ready: bool, + compute_ready: bool, + render_broken: bool, +} + +/// Retain submit-time GEM and timestamp bindings until actual completion. +/// Command preparation never reads caller CDM or shader resource data. +struct Bindings { + _mappings: KVec, + _timestamps: KVec, +} + +struct FileState { + next_vm: u32, + next_queue: u32, + next_timestamp: u32, + queues: KVec, + timestamps: KVec, + retained_timestamps: KVec, + vms: KVec, +} + +struct Queue { + id: u32, + vm: u32, + entity: sched::Entity, + firmware: Arc>, +} +#[derive(Clone)] +struct TimestampBuffer { + id: u32, + sg: Arc>, + base: u64, + extent: u64, + address: u64, + size: u64, +} + +#[pin_data] +pub(crate) struct Data { + gpu_revision: u32, + core_mask: u32, + maximum_frequency_khz: u32, + scheduler: sched::Scheduler, + #[pin] + admission: Mutex>, + failed: AtomicBool, + notifications: Arc, + #[pin] + runtime: Mutex, + #[pin] + engine: Mutex, +} + +#[vtable] +impl drm::driver::Driver for Driver { + type Data = Data; + type File = File; + type Object = Object; + const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c_str!("asahi"), + desc: c_str!("Apple G16 Graphics"), + }; + const FEATURES: u32 = drm::driver::FEAT_GEM + | drm::driver::FEAT_RENDER + | drm::driver::FEAT_SYNCOBJ + | drm::driver::FEAT_SYNCOBJ_TIMELINE; + + kernel::declare_drm_ioctls! { + (ASAHI_GET_PARAMS, drm_asahi_get_params, ioctl::RENDER_ALLOW, File::get_params), + (ASAHI_GET_TIME, drm_asahi_get_time, ioctl::RENDER_ALLOW, File::get_time), + (ASAHI_VM_CREATE, drm_asahi_vm_create, ioctl::RENDER_ALLOW, File::vm_create), + (ASAHI_VM_DESTROY, drm_asahi_vm_destroy, ioctl::RENDER_ALLOW, File::vm_destroy), + (ASAHI_VM_BIND, drm_asahi_vm_bind, ioctl::RENDER_ALLOW, File::vm_bind), + (ASAHI_GEM_CREATE, drm_asahi_gem_create, ioctl::RENDER_ALLOW, File::gem_create), + (ASAHI_GEM_MMAP_OFFSET, drm_asahi_gem_mmap_offset, ioctl::RENDER_ALLOW, File::gem_mmap_offset), + (ASAHI_GEM_BIND_OBJECT, drm_asahi_gem_bind_object, ioctl::RENDER_ALLOW, File::bind_object), + (ASAHI_QUEUE_CREATE, drm_asahi_queue_create, ioctl::RENDER_ALLOW, File::queue_create), + (ASAHI_QUEUE_DESTROY, drm_asahi_queue_destroy, ioctl::RENDER_ALLOW, File::queue_destroy), + (ASAHI_SUBMIT, drm_asahi_submit, ioctl::RENDER_ALLOW, File::submit), + } +} + +pub(crate) fn register(pdev: &platform::Device) -> Result { + use kernel::dma::{Device as _, DmaMask}; + // SAFETY: G16's hardware output address width was validated at bootstrap. + unsafe { + pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)?; + } + let runtime = crate::g16::Bootstrap::new(pdev)?; + let data = try_pin_init!(Data { + gpu_revision: runtime.gpu_revision, + core_mask: runtime.core_mask, maximum_frequency_khz: runtime.maximum_frequency_khz, + scheduler: sched::Scheduler::new(pdev.as_ref(), 4, 128, 0, 30000, c_str!("asahi_m4"))?, + admission <- kernel::new_mutex!(KVec::new()), + failed: AtomicBool::new(false), + notifications: runtime.notifications.clone(), + runtime <- kernel::new_mutex!(runtime), + engine <- kernel::new_mutex!(Engine { running: false, cursor: 0, jobs: KVec::new(), issued: KVec::new() }), }); + let device = Device::new(pdev.as_ref(), data)?; + drm::driver::Registration::new_foreign_owned(&device, pdev.as_ref(), 0)?; + Ok(device) +} + +pub(crate) struct Buffer { + owner: u64, + vm: u32, +} +#[vtable] +impl DriverObject for Buffer { + type Driver = Driver; + type Args = (u64, u32); + fn new(_dev: &Device, _size: usize, args: Self::Args) -> impl PinInit { + Ok(Self { + owner: args.0, + vm: args.1, + }) + } + + fn export(obj: &Object, flags: u32) -> Result> { + if obj.vm != 0 { + return Err(EINVAL); + } + obj.prime_export(flags) + } + + fn close(obj: &Object, file: &DrmFile) { + // Mesa may reuse this virtual address after GEM_CLOSE. Hold the file + // state against new submissions, and retire only VMs using this BO. + // Workers never acquire file state while completing their fences. + let inner = file.inner(); + if inner.device.failed.load(Ordering::Acquire) { + return; + } + let result: Result = (|| { + let mut state = inner.state.lock(); + for vm in &mut state.vms { + if !vm.mappings.iter().any(|m| core::ptr::eq(m.sg.object(), obj)) { + continue; + } + drain(&mut vm.pending)?; + let runtime = inner.device.runtime.lock(); + while let Some(range) = vm.mappings.iter() + .find(|m| core::ptr::eq(m.sg.object(), obj)) + .map(|m| m.range.clone()) + { + vm.unmap(range, &runtime)?; + } + } + Ok(()) + })(); + if let Err(error) = result { + // close cannot report failure. Preserve backing for late DMA and + // reject further submissions if completion was not established. + inner.device.failed.store(true, Ordering::Release); + pr_err!("G16: failed to unbind closed GEM object: {:?}\n", error); + } + } +} + +#[pin_data(PinnedDrop)] +pub(crate) struct File { + device: DeviceRef, + raw: AtomicPtr, + id: u64, + state: Arc>, +} +impl drm::file::DriverFile for File { + type Driver = Driver; + fn open(device: &Device) -> Result>> { + KBox::pin_init( + try_pin_init!(Self { + device: device.into(), + raw: AtomicPtr::new(core::ptr::null_mut()), + id: NEXT_FILE.fetch_add(1, Ordering::Relaxed), + state: Arc::pin_init( + kernel::new_mutex!(FileState { + next_vm: 1, + next_queue: 1, + next_timestamp: 1, + queues: KVec::new(), + timestamps: KVec::new(), + retained_timestamps: KVec::new(), + vms: KVec::new() + }), + GFP_KERNEL + )?, + }), + GFP_KERNEL, + ) + } + fn post_open(&self, file: &DrmFile) { + self.raw.store(file.as_raw(), Ordering::Release); + } + fn as_raw(&self) -> *mut bindings::drm_file { + self.raw.load(Ordering::Acquire) + } +} + +#[pinned_drop] +impl PinnedDrop for File { + fn drop(self: Pin<&mut Self>) { + // Entity destruction cancels dependency-blocked jobs. Never hold the + // state lock while DRM waits for scheduler callbacks. + let queues = core::mem::take(&mut self.state.lock().queues); + drop(queues); + let mut admission = self.device.admission.lock(); + if drain(&mut admission).is_err() { + // Completion was not established. Retain roots and every pinned + // GEM page until reboot; do not let late DMA reach reused RAM. + core::mem::forget(self.state.clone()); + self.device.failed.store(true, Ordering::Release); + return; + } + let state = self.state.lock(); + let mut runtime = self.device.runtime.lock(); + for vm in state.vms.iter().rev() { + if vm.detach(&mut runtime).is_err() { + // Retain GPU-visible mappings if the installed root cannot + // be detached safely; device reset releases the hardware. + drop(runtime); + drop(state); + core::mem::forget(self.state.clone()); + return; + } + } + for timestamp in &state.retained_timestamps { + if runtime + .release_timestamp(timestamp.base..timestamp.base + timestamp.extent) + .is_err() + { + core::mem::forget(self.state.clone()); + break; + } + } + } +} + +impl File { + fn get_time( + _device: &Device, + data: &mut uapi::drm_asahi_get_time, + _file: &DrmFile, + ) -> Result { + if data.flags != 0 { + return Err(EINVAL); + } + let (counter, frequency): (u64, u64); + // SAFETY: These instructions only read architectural timer registers. + unsafe { + core::arch::asm!("mrs {c}, cntpct_el0", "mrs {f}, cntfrq_el0", + c = out(reg) counter, f = out(reg) frequency); + } + if frequency == 0 { + return Err(EIO); + } + data.gpu_timestamp = + counter / frequency * 1_000_000_000 + (counter % frequency) * 1_000_000_000 / frequency; + Ok(0) + } + + fn vm_create( + device: &Device, + data: &mut uapi::drm_asahi_vm_create, + file: &DrmFile, + ) -> Result { + if data.pad != 0 + || data.kernel_start < 0x4000 + || data.kernel_end > VM_END + || data + .kernel_end + .checked_sub(data.kernel_start) + .ok_or(EINVAL)? + < 0x20000000 + || (data.kernel_start | data.kernel_end) & 0x3fff != 0 + { + return Err(EINVAL); + } + let inner = file.inner(); + let mut state = inner.state.lock(); + let id = state.next_vm; + let next = id.checked_add(1).ok_or(ENOSPC)?; + let reservation = Object::new( + device, + 0x4000, + shmem::ObjectConfig { + map_wc: false, + parent_resv_obj: None, + }, + (inner.id, id), + )?; + state.vms.push( + Vm { + id, + visible: true, + kernel: data.kernel_start..data.kernel_end, + space: Arc::pin_init( + kernel::new_mutex!(crate::g16_vm::AddressSpace::new()?), + GFP_KERNEL, + )?, + compute_space: Arc::pin_init( + kernel::new_mutex!(crate::g16_vm::AddressSpace::new()?), + GFP_KERNEL, + )?, + mappings: KVec::new(), + pending: KVec::new(), + reservation, + render_support: None, + render_ready: false, + compute_ready: false, + render_broken: false, + }, + GFP_KERNEL, + )?; + state.next_vm = next; + data.vm_id = id; + Ok(0) + } + + fn vm_destroy( + device: &Device, + data: &mut uapi::drm_asahi_vm_destroy, + file: &DrmFile, + ) -> Result { + if data.pad != 0 { + return Err(EINVAL); + } + let mut admission = device.admission.lock(); + let inner = file.inner(); + let mut state = inner.state.lock(); + let index = state + .vms + .iter() + .position(|v| v.id == data.vm_id && v.visible) + .ok_or(ENOENT)?; + if state.queues.iter().any(|q| q.vm == data.vm_id) { + // Removing the public handle must not invalidate existing queues. + // They retain this VM until the last queue is destroyed. + state.vms[index].visible = false; + } else { + drain(&mut admission)?; + state.vms[index].detach(&mut device.runtime.lock())?; + state.vms.swap_remove(index); + } + Ok(0) + } + + fn get_params( + device: &Device, + data: &mut uapi::drm_asahi_get_params, + _file: &DrmFile, + ) -> Result { + if data.param_group != 0 || data.pad != 0 { + return Err(EINVAL); + } + let mut params = uapi::drm_asahi_params_global { + features: 0, + gpu_generation: 16, + gpu_variant: b'G' as u32, + gpu_revision: device.gpu_revision, + chip_id: 0x8132, + num_dies: 1, + num_clusters_total: 1, + num_cores_per_cluster: 10, + max_frequency_khz: device.maximum_frequency_khz, + core_masks: [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize], + vm_start: 0x4000, + vm_end: (1 << 42) - 0x8000, + vm_kernel_min_size: 0x20000000, + max_commands_per_submission: 64, + max_attachments: 16, + command_timestamp_frequency_hz: 1_000_000_000, + }; + params.core_masks[0] = u64::from(device.core_mask); + let size = core::mem::size_of_val(¶ms).min(data.size.try_into()?); + let mut writer = UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); + // SAFETY: The UAPI object has fully initialized fields, and size is + // bounded by its extent. This copies once into the caller's buffer. + writer.write_slice(unsafe { + core::slice::from_raw_parts( + (¶ms as *const uapi::drm_asahi_params_global).cast(), + size, + ) + })?; + Ok(0) + } + + fn queue_create( + device: &Device, + data: &mut uapi::drm_asahi_queue_create, + file: &DrmFile, + ) -> Result { + if data.flags != 0 || data.priority > 1 || data.usc_exec_base != 1 << 40 { + return Err(EINVAL); + } + let inner = file.inner(); + let mut state = inner.state.lock(); + if !state.vms.iter().any(|vm| vm.id == data.vm_id && vm.visible) { + return Err(ENOENT); + } + let id = state.next_queue; + let next = id.checked_add(1).ok_or(ENOSPC)?; + let priority = if data.priority == 0 { + sched::Priority::Low + } else { + sched::Priority::Normal + }; + let entity = sched::Entity::new(&device.scheduler, priority)?; + state.queues.push( + Queue { + firmware: Arc::pin_init( + // Firmware priority runs in the opposite direction: + // public low/medium select native classes 3/2. + kernel::new_mutex!(crate::g16::FirmwareQueues::new(3 - data.priority)), + GFP_KERNEL, + )?, + id, + vm: data.vm_id, + entity, + }, + GFP_KERNEL, + )?; + state.next_queue = next; + data.queue_id = id; + Ok(0) + } + + fn queue_destroy( + device: &Device, + data: &mut uapi::drm_asahi_queue_destroy, + file: &DrmFile, + ) -> Result { + if data.pad != 0 { + return Err(EINVAL); + } + let mut admission = device.admission.lock(); + let inner = file.inner(); + let queue = { + let mut state = inner.state.lock(); + let index = state + .queues + .iter() + .position(|q| q.id == data.queue_id) + .ok_or(ENOENT)?; + state.queues.swap_remove(index) + }; + // Cancel dependency-blocked jobs before waiting on their lifetime + // fences. Scheduler destruction must run outside the file-state lock. + let vm_id = queue.vm; + drop(queue); + drain(&mut admission)?; + let mut state = inner.state.lock(); + if !state.queues.iter().any(|q| q.vm == vm_id) { + if let Some(index) = state.vms.iter().position(|v| v.id == vm_id && !v.visible) { + state.vms[index].detach(&mut device.runtime.lock())?; + state.vms.swap_remove(index); + } + } + Ok(0) + } + + fn bind_object( + device: &Device, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + if data.pad != 0 || data.vm_id != 0 { + return Err(EINVAL); + } + let mut admission = device.admission.lock(); + drain(&mut admission)?; + let inner = file.inner(); + let mut state = inner.state.lock(); + match data.op { + uapi::drm_asahi_bind_object_op_DRM_ASAHI_BIND_OBJECT_OP_BIND => { + if data.flags + != uapi::drm_asahi_bind_object_flags_DRM_ASAHI_BIND_OBJECT_USAGE_TIMESTAMPS + || data.range == 0 + || (data.offset | data.range) & 0x3fff != 0 + { + return Err(EINVAL); + } + let bo = Object::lookup_handle(file, data.handle)?; + if data.offset.checked_add(data.range).ok_or(EINVAL)? > bo.size() as u64 { + return Err(EINVAL); + } + let id = state.next_timestamp; + let next = id.checked_add(1).ok_or(ENOSPC)?; + let sg = Arc::new(bo.owned_sg_table()?, GFP_KERNEL)?; + let first = data.offset & !0x3fff; + let extent = ((data.offset + data.range + 0x3fff) & !0x3fff) - first; + state.timestamps.reserve(1, GFP_KERNEL)?; + state.retained_timestamps.reserve(1, GFP_KERNEL)?; + let mut runtime = device.runtime.lock(); + let base = runtime.reserve_timestamp(extent)?; + let timestamp = TimestampBuffer { + id, + sg, + base, + extent, + address: base + (data.offset - first), + size: data.range, + }; + // Keep backing even if mapping installation or rollback fails. + state + .retained_timestamps + .push(timestamp.clone(), GFP_KERNEL)?; + let mut offset = first; + let mut address = base; + let result = (|| { + for segment in timestamp.sg.iter() { + let length = u64::from(segment.dma_len()); + if offset >= length { + offset -= length; + continue; + } + let size = (length - offset).min(base + extent - address); + runtime.map_timestamp( + address, + segment.dma_address() + offset, + size.try_into()?, + )?; + address += size; + offset = 0; + if address == base + extent { + break; + } + } + if address != base + extent { + return Err(EIO); + } + Ok(()) + })(); + if let Err(error) = result { + runtime.release_timestamp(base..base + extent)?; + state.retained_timestamps.pop(); + return Err(error); + } + state.timestamps.push(timestamp, GFP_KERNEL)?; + state.next_timestamp = next; + data.object_handle = id; + } + uapi::drm_asahi_bind_object_op_DRM_ASAHI_BIND_OBJECT_OP_UNBIND => { + if data.flags != 0 || data.offset != 0 || data.range != 0 || data.handle != 0 { + return Err(EINVAL); + } + let index = state + .timestamps + .iter() + .position(|t| t.id == data.object_handle) + .ok_or(ENOENT)?; + state.timestamps.swap_remove(index); + } + _ => return Err(EINVAL), + } + Ok(0) + } + + fn vm_bind(device: &Device, data: &mut uapi::drm_asahi_vm_bind, file: &DrmFile) -> Result { + if data.pad != 0 || data.num_binds > 4096 || data.stride == 0 { + return Err(EINVAL); + } + let byte_count = (data.stride as usize) + .checked_mul(data.num_binds as usize) + .ok_or(EINVAL)?; + if byte_count > 1024 * 1024 { + return Err(EINVAL); + } + let mut records = KVec::new(); + UserSlice::new(UserPtr::from_addr(data.userptr.try_into()?), byte_count) + .reader() + .read_all(&mut records, GFP_KERNEL)?; + let mut reader = crate::util::Reader::new(&records); + if device.failed.load(Ordering::Acquire) { + return Err(EIO); + } + let inner = file.inner(); + let mut state = inner.state.lock(); + let vm = state + .vms + .iter_mut() + .find(|vm| vm.id == data.vm_id && vm.visible) + .ok_or(ENOENT)?; + for _ in 0..data.num_binds { + let bind: uapi::drm_asahi_gem_bind_op = reader.read_up_to(data.stride as usize)?; + let (flags, handle, offset, size, address) = + (bind.flags, bind.handle, bind.offset, bind.range, bind.addr); + let end = address.checked_add(size).ok_or(EINVAL)?; + if size == 0 + || (address | size | offset) & 0x3fff != 0 + || address < 0x4000 + || end > VM_END + || (address < vm.kernel.end && end > vm.kernel.start) + || (address < 0x73_0000_0000 && end > 0x71_0000_0000) + || [ + (0x7000000000, 0x7000400000), + (0x7000408000, 0x7000418000), + (0x7000420000, 0x7002cb8000), + (0x7003108000, 0x7005180000), + ] + .iter() + .any(|(base, limit)| address < *limit && end > *base) + || compute::private_ranges() + .iter() + .any(|(base, len)| address < base + *len as u64 && end > *base) + || render::fixed_ranges() + .iter() + .any(|(base, len)| address < base + *len as u64 && end > *base) + { + return Err(EINVAL); + } + if flags == uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND { + if handle != 0 || offset != 0 { + return Err(EINVAL); + } + // Only this VM can reference the removed mapping. Fresh + // binds never wait for hardware, including on this VM. + drain(&mut vm.pending)?; + let runtime = device.runtime.lock(); + vm.unmap(address..end, &runtime)?; + continue; + } + let read = flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0; + let write = flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0; + let single = flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE != 0; + if flags & !0xe != 0 || (!read && !write) { + return Err(EINVAL); + } + let bo = Object::lookup_handle(file, handle)?; + if bo.vm != 0 && (bo.vm != vm.id || bo.owner != inner.id) { + return Err(EINVAL); + } + if offset + .checked_add(if single { 0x4000 } else { size }) + .ok_or(EINVAL)? + > bo.size() as u64 + { + return Err(EINVAL); + } + let access = match (read, write) { + (true, true) => crate::pgtable::prot::PROT_GPU_SHARED_RW, + (true, false) => crate::pgtable::prot::PROT_GPU_SHARED_RO, + _ => crate::pgtable::prot::PROT_GPU_SHARED_WO, + }; + let runtime = device.runtime.lock(); + vm.map( + address..end, + offset, + bo.owned_sg_table()?, + access, + flags & 6, + single, + &runtime, + )?; + } + Ok(0) + } + + fn gem_create( + device: &Device, + data: &mut uapi::drm_asahi_gem_create, + file: &DrmFile, + ) -> Result { + let private = data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0; + if data.flags + & !(uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK + | uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE) + != 0 + || (!private && data.vm_id != 0) + || data.size == 0 + || data.pad != 0 + { + return Err(EINVAL); + } + let size: usize = data.size.try_into()?; + let size = size.checked_add(0x3fff).ok_or(EINVAL)? & !0x3fff; + let inner = file.inner(); + let state = inner.state.lock(); + let parent = if private { + Some( + &*state + .vms + .iter() + .find(|vm| vm.id == data.vm_id && vm.visible) + .ok_or(ENOENT)? + .reservation, + ) + } else { + None + }; + let bo = Object::new( + device, + size, + shmem::ObjectConfig { + map_wc: data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, + parent_resv_obj: parent, + }, + (inner.id, if private { data.vm_id } else { 0 }), + )?; + data.handle = bo.create_handle(file)?; + Ok(0) + } + + fn gem_mmap_offset( + _device: &Device, + data: &mut uapi::drm_asahi_gem_mmap_offset, + file: &DrmFile, + ) -> Result { + if data.flags != 0 { + return Err(EINVAL); + } + data.offset = Object::lookup_handle(file, data.handle)?.create_mmap_offset()?; + Ok(0) + } +} + +impl Vm { + fn detach(&self, runtime: &mut crate::g16::Bootstrap) -> Result { + runtime.release_render_vm(self.space.lock().roots())?; + let compute = self.compute_space.lock(); + runtime.release_render_vm(compute.roots())?; + Ok(()) + } + + fn map( + &mut self, + range: Range, + mut offset: u64, + sg: shmem::SGTable, + access: crate::pgtable::Prot, + permissions: u32, + single: bool, + runtime: &crate::g16::Bootstrap, + ) -> Result { + if self + .mappings + .iter() + .any(|m| range.start < m.range.end && range.end > m.range.start) + { + return Err(EEXIST); + } + let bo_offset = offset; + let sg = Arc::new(sg, GFP_KERNEL)?; + let mut chunks = KVec::new(); + let mut va = range.start; + for segment in sg.iter() { + let length = u64::from(segment.dma_len()); + if offset >= length { + offset -= length; + continue; + } + let pa = segment.dma_address().checked_add(offset).ok_or(EINVAL)?; + let length = (length - offset).min(range.end - va); + offset = 0; + if (va | pa | length) & 0x3fff != 0 { + return Err(EINVAL); + } + let end = if single { range.end } else { va + length }; + chunks.push((va, end, pa), GFP_KERNEL)?; + va = end; + if va == range.end { + break; + } + } + if va != range.end { + return Err(EINVAL); + } + self.mappings.push( + Mapping { + range: range.clone(), + permissions, + offset: bo_offset, + single, + sg, + }, + GFP_KERNEL, + )?; + let result = (|| { + // The shim uses distinct roots for the fixed render and compute + // private namespaces. Both views share the caller's GEM pages. + for view in [&self.space, &self.compute_space] { + let mut space = view.lock(); + for &(start, end, pa) in &chunks { + space.low.map_pages(start..end, pa, access, single)?; + let last = if single { + pa + } else { + pa + end - start - 0x4000 + }; + if space.low.translate(start)? != Some(pa) + || space.low.translate(end - 0x4000)? != Some(last) + { + return Err(EIO); + } + } + } + Ok(()) + })(); + if let Err(e) = result { + // Keep the backing pinned if rollback itself cannot finish. + self.space.lock().low.unmap_pages(range.clone())?; + self.compute_space.lock().low.unmap_pages(range)?; + self.publish_mappings(runtime); + self.mappings.pop(); + return Err(e); + } + self.publish_mappings(runtime); + Ok(()) + } + + /// Publish translation changes before another job can use this VM. + fn publish_mappings(&self, runtime: &crate::g16::Bootstrap) { + for view in [&self.space, &self.compute_space] { + let space = view.lock(); + space.sync(); + runtime.invalidate_client(&space); + } + } + + fn unmap(&mut self, range: Range, runtime: &crate::g16::Bootstrap) -> Result { + // Callers drain this VM's jobs and hold file state against new uses. + // A failure may have signaled those fences while we waited. The + // runtime lock prevents a concurrent failure during this teardown. + if runtime.render_failed() { + return Err(EIO); + } + // Prepare ownership splits before changing any page-table entry. + let mut retained = KVec::new(); + for m in &self.mappings { + for part in [ + m.range.start..m.range.end.min(range.start), + m.range.start.max(range.end)..m.range.end, + ] { + if part.start < part.end { + retained.push( + Mapping { + offset: m.offset + + if m.single { + 0 + } else { + part.start - m.range.start + }, + range: part, + permissions: m.permissions, + single: m.single, + sg: m.sg.clone(), + }, + GFP_KERNEL, + )?; + } + } + } + self.space.lock().low.reserve_invalidation()?; + self.compute_space.lock().low.reserve_invalidation()?; + self.space.lock().low.unmap_pages(range.clone())?; + self.compute_space.lock().low.unmap_pages(range.clone())?; + self.publish_mappings(runtime); + // Both views are unmapped and their translation invalidations have + // completed. Drop removed backing now; surviving partial mappings + // retain their SGTable references. Any earlier error leaves the + // original ownership in self.mappings, including for partial unmaps. + self.mappings = retained; + Ok(()) + } +} + +static FENCE_KEY: Pin<&kernel::sync::LockClassKey> = kernel::static_lock_class!(); + +/// Wait only for actual worker completion, including on scheduler timeout. +/// Callers hold admission or the affected file state, preventing new uses +/// of the mappings protected by these fences. +fn drain(pending: &mut KVec) -> Result { + for fence in pending.iter() { + // SAFETY: Each entry owns its fence reference throughout the wait. + let result = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(35000) as _, + ) + }; + if result < 0 { + return Err(Error::from_errno(result as i32)); + } + if result == 0 { + return Err(ETIMEDOUT); + } + } + pending.clear(); + Ok(()) +} + +/// Owns copied commands and the VM storage until hardware completion. The +/// worker never acquires admission or the file-state lock, so another SUBMIT +/// can publish pending fences while the current job executes. +#[pin_data(PinnedDrop)] +struct Execution { + id: u64, + firmware: Arc>, + // UAPI subqueue indices, resolved against previous submissions and this batch. + barriers: KVec<[u16; 2]>, + device: DeviceRef, + _owner: Arc>, + render_space: Arc>, + space: Arc>, + _bindings: Bindings, + support: Option>, + parameters: KVec, + timestamps: KVec<[u64; 4]>, + completion: dma_fence::Fence, + started: AtomicBool, + #[pin] + work: Work, +} + +impl_has_work! { impl HasWork for Execution { self.work } } + +struct Running { + execution: Arc, + next: usize, + pending: usize, + error: Option, + history: [KVec>; 2], +} + +fn resolve_barriers( + history: &[KVec>; 2], + barriers: [u16; 2], +) -> Result<[Option; 2]> { + let mut dependencies = [None; 2]; + for (stage, barrier) in barriers.into_iter().enumerate() { + if barrier != 0xffff { + dependencies[stage] = *history[stage].get(barrier as usize).ok_or(EIO)?; + } + } + Ok(dependencies) +} +struct Issued { + id: u64, + owner: u64, +} +struct Engine { + running: bool, + cursor: usize, + jobs: KVec, + issued: KVec, +} + +impl WorkItem for Execution { + type Pointer = Arc; + + fn run(this: Arc) { + // Exactly one pump owns publication order. It holds these locks only + // while preparing/publishing or inspecting retirement, never asleep. + // Scheduler callbacks append ready jobs even while hardware is busy. + let mut wake_result = Ok(()); + loop { + let generation = this.device.notifications.snapshot(); + let mut engine = this.device.engine.lock(); + let mut runtime = this.device.runtime.lock(); + let result = (|| -> Result { + wake_result?; + generation?; + if this.device.failed.load(Ordering::Acquire) { + return Err(EIO); + } + let mut progress = false; + while let Some((id, result)) = runtime.retire().inspect_err(|error| { + pr_err!("G16: firmware retirement failed: {:?}\n", error); + })? { + let index = engine.issued.iter().position(|r| r.id == id).ok_or(EIO)?; + let issued = engine.issued.remove(index).map_err(|_| EIO)?; + let job = engine + .jobs + .iter_mut() + .find(|r| r.execution.id == issued.owner) + .ok_or(EIO)?; + job.pending -= 1; + if let Err(error) = result { + job.error = Some(error); + } + progress = true; + } + let mut admitted = 0; + loop { + let previous = admitted; + let start = engine.cursor; + for offset in 0..engine.jobs.len() { + let index = (start + offset) % engine.jobs.len(); + // Preserve submit-relative barrier index zero: all earlier + // submissions on this public queue must have published. + if engine.jobs[..index].iter().any(|earlier| { + earlier.error.is_none() + && earlier.next < earlier.execution.parameters.len() + && Arc::ptr_eq( + &earlier.execution.firmware, + &engine.jobs[index].execution.firmware, + ) + }) { + continue; + } + let job = &mut engine.jobs[index]; + if job.error.is_some() || job.next == job.execution.parameters.len() { + continue; + } + if job.history[0].is_empty() { + let previous = job.execution.firmware.lock().last; + for (history, stamp) in job.history.iter_mut().zip(previous) { + history.push(stamp, GFP_KERNEL)?; + } + } + let dependencies = + resolve_barriers(&job.history, job.execution.barriers[job.next])?; + let execution = job.execution.clone(); + let compute = + matches!(execution.parameters[job.next], Parameters::Compute(_)); + // Intra-queue UAPI dependencies are encoded into firmware + // rings; preparation does not read GPU-produced data. + engine.issued.reserve(1, GFP_KERNEL)?; + let job = &mut engine.jobs[index]; + crate::mem::sync(); + let receipt = match &execution.parameters[job.next] { + Parameters::Render(p) => runtime.submit_render( + execution.firmware.clone(), + execution.render_space.clone(), + p, + execution.support.as_deref().ok_or(EIO)?, + execution.timestamps[job.next], + &dependencies, + ), + Parameters::Compute(p) => runtime.submit_compute( + execution.firmware.clone(), + execution.space.clone(), + p, + execution.timestamps[job.next], + &dependencies, + ), + }; + let receipt = match receipt { + Ok(receipt) => receipt, + Err(EBUSY) if !runtime.render_failed() => continue, + Err(error) if !runtime.render_failed() => { + pr_err!( + "G16: command {} preparation failed: {:?}\n", + job.next, + error + ); + job.error = Some(error); + progress = true; + continue; + } + Err(error) => return Err(error), + }; + job.history[usize::from(compute)].push(Some(receipt.stamp), GFP_KERNEL)?; + let issued = Issued { + id: receipt.id, + owner: execution.id, + }; + job.next += 1; + job.pending += 1; + engine.issued.push(issued, GFP_KERNEL)?; + // Resume after the last admitted job when a scarce slot + // becomes free, rather than repeatedly favoring index zero. + engine.cursor = index + 1; + progress = true; + admitted += 1; + if admitted == 64 { + break; + } + } + if admitted == previous || admitted == 64 { + break; + } + } + runtime.flush()?; + Ok(progress) + })(); + if let Err(error) = result { + pr_err!( + "G16: submission pipeline failed: {:?}, {} jobs, {} issued\n", + error, + engine.jobs.len(), + engine.issued.len() + ); + runtime.poison(); + this.device.failed.store(true, Ordering::Release); + let jobs = core::mem::take(&mut engine.jobs); + engine.issued.clear(); + engine.running = false; + drop(runtime); + drop(engine); + // Quarantined context roots remain installed and retained; + // a failed fence never authorizes reuse of their backing. + for job in jobs { + job.execution.completion.set_error(error); + let _ = job.execution.completion.signal(); + } + return; + } + if let Some(index) = engine.jobs.iter().position(|j| { + j.pending == 0 && (j.error.is_some() || j.next == j.execution.parameters.len()) + }) { + let job = engine + .jobs + .remove(index) + .expect("index found under engine lock"); + drop(runtime); + drop(engine); + if let Some(error) = job.error { + job.execution.completion.set_error(error); + } + let _ = job.execution.completion.signal(); + continue; + } + if engine.jobs.is_empty() { + engine.running = false; + return; + } + let timeout_ms = runtime.watchdog_remaining_ms(); + drop(runtime); + drop(engine); + if !result.unwrap_or(false) { + wake_result = this.device.notifications.wait( + generation.expect("checked before servicing jobs"), + timeout_ms, + ); + } + } + } +} + +#[pinned_drop] +impl PinnedDrop for Execution { + fn drop(self: Pin<&mut Self>) { + // An entity can discard a job still waiting for its dependencies. + // Such a job never published to firmware, but its fence must finish. + if !self.started.load(Ordering::Acquire) { + self.completion.set_error(ECANCELED); + let _ = self.completion.signal(); + } + } +} + +struct SubmissionJob { + execution: Arc, +} + +impl sched::JobImpl for SubmissionJob { + fn run(job: &mut sched::Job) -> Result> { + let execution = &job.execution; + // Firmware jobs cannot be replayed after a fault without a reboot. + if execution.started.swap(true, Ordering::AcqRel) { + return Err(EIO); + } + let fence = execution.completion.clone(); + let mut engine = execution.device.engine.lock(); + let result = (|| -> Result { + let mut history = [KVec::new(), KVec::new()]; + for events in &mut history { + events.reserve(65, GFP_KERNEL)?; + } + engine.jobs.push( + Running { + execution: execution.clone(), + next: 0, + pending: 0, + error: None, + history, + }, + GFP_KERNEL, + )?; + Ok(()) + })(); + if let Err(error) = result { + execution.completion.set_error(error); + let _ = execution.completion.signal(); + return Err(error); + } + if !engine.running { + engine.running = true; + let _ = workqueue::system_unbound().enqueue(execution.clone()); + } + // A running worker may be asleep waiting for firmware on another + // queue. New ready work must wake it even if no GPU event arrives. + execution.device.notifications.notify(false); + Ok(Some(fence)) + } + + fn timed_out(job: &mut sched::Job) -> sched::Status { + job.execution.device.failed.store(true, Ordering::Release); + job.execution.device.notifications.notify(false); + // Wake the worker to quarantine firmware mappings and fail the jobs. + // Do not signal its lifetime fence before that cleanup here. + sched::Status::NoDevice + } + + fn cancel(job: &mut sched::Job) { + if !job.execution.started.load(Ordering::Acquire) { + job.execution.completion.set_error(ECANCELED); + let _ = job.execution.completion.signal(); + } + } +} + +struct Completion; +#[vtable] +impl dma_fence::FenceOps for Completion { + fn get_driver_name<'a>(self: &'a dma_fence::FenceObject) -> &'a kernel::str::CStr { + c_str!("asahi") + } + fn get_timeline_name<'a>(self: &'a dma_fence::FenceObject) -> &'a kernel::str::CStr { + c_str!("m4-execution") + } +} +struct OutputSync { + object: drm::syncobj::SyncObj, + chain: Option, + point: u64, +} + +impl File { + fn submit(device: &Device, data: &mut uapi::drm_asahi_submit, file: &DrmFile) -> Result { + if data.flags != 0 + || data.pad != 0 + || data.cmdbuf_size == 0 + || data.cmdbuf_size > 1024 * 1024 + || data.in_sync_count > 4096 + || data.out_sync_count > 4096 + { + return Err(EINVAL); + } + let mut bytes = KVec::new(); + UserSlice::new( + UserPtr::from_addr(data.cmdbuf.try_into()?), + data.cmdbuf_size as usize, + ) + .reader() + .read_all(&mut bytes, GFP_KERNEL)?; + let mut commands = KVec::new(); + let mut barriers = KVec::new(); + let mut attachments = KVec::new(); + let mut offset = 0usize; + let mut renders = 0u16; + let mut computes = 0u16; + while offset < bytes.len() { + let mut reader = crate::util::Reader::new(&bytes[offset..]); + let header: uapi::drm_asahi_cmd_header = reader.read()?; + offset = offset.checked_add(8).ok_or(EINVAL)?; + let end = offset.checked_add(header.size as usize).ok_or(EINVAL)?; + let payload = bytes.get(offset..end).ok_or(EINVAL)?; + offset = end; + match u32::from(header.cmd_type) { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + if (header.vdm_barrier != 0xffff && header.vdm_barrier > renders) + || (header.cdm_barrier != 0xffff && header.cdm_barrier > computes) + || renders + computes == 64 + { + return Err(EINVAL); + } + let command = crate::util::Reader::new(payload) + .read_up_to::(payload.len())?; + commands.push(Command::Render(command), GFP_KERNEL)?; + barriers.push([header.vdm_barrier, header.cdm_barrier], GFP_KERNEL)?; + renders += 1; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + if (header.vdm_barrier != 0xffff && header.vdm_barrier > renders) + || (header.cdm_barrier != 0xffff && header.cdm_barrier > computes) + || renders + computes == 64 + { + return Err(EINVAL); + } + let command = crate::util::Reader::new(payload) + .read_up_to::(payload.len())?; + commands.push(Command::Compute(command), GFP_KERNEL)?; + barriers.push([header.vdm_barrier, header.cdm_barrier], GFP_KERNEL)?; + computes += 1; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS + | uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS + | uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS => { + if header.vdm_barrier != 0xffff + || header.cdm_barrier != 0xffff + || payload.len() % 24 != 0 + || payload.len() / 24 > 16 + { + return Err(EINVAL); + } + let mut reader = crate::util::Reader::new(payload); + while !reader.is_empty() { + let attachment: uapi::drm_asahi_attachment = reader.read()?; + if attachment.flags != 0 + || attachment.pad != 0 + || attachment.pointer.checked_add(attachment.size).is_none() + { + return Err(EINVAL); + } + attachments.push(attachment, GFP_KERNEL)?; + } + } + _ => return Err(ENOTSUPP), + } + } + if renders + computes == 0 { + return Err(EINVAL); + } + let count = data.in_sync_count + data.out_sync_count; + let mut sync_bytes = KVec::new(); + UserSlice::new( + UserPtr::from_addr(data.syncs.try_into()?), + count as usize * 16, + ) + .reader() + .read_all(&mut sync_bytes, GFP_KERNEL)?; + let mut outputs = KVec::new(); + let mut dependencies = KVec::new(); + for (index, bytes) in sync_bytes.chunks_exact(16).enumerate() { + let kind = u32::from_ne_bytes(bytes[0..4].try_into().map_err(|_| EINVAL)?); + let handle = u32::from_ne_bytes(bytes[4..8].try_into().map_err(|_| EINVAL)?); + let point = u64::from_ne_bytes(bytes[8..16].try_into().map_err(|_| EINVAL)?); + if kind > 1 || (kind == 0 && point != 0) { + return Err(EINVAL); + } + let object = drm::syncobj::SyncObj::lookup_handle(file, handle)?; + if index >= data.in_sync_count as usize { + outputs.push( + OutputSync { + object, + point, + chain: if kind == 1 { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, + }, + GFP_KERNEL, + )?; + } else { + let fence = object.fence_get().ok_or(EINVAL)?; + let fence = if kind == 1 { + fence.chain_find_seqno(point)? + } else { + Some(fence) + }; + if let Some(fence) = fence { + dependencies.push(fence, GFP_KERNEL)?; + } + } + } + let inner = file.inner(); + let mut admission = device.admission.lock(); + if device.failed.load(Ordering::Acquire) { + return Err(EIO); + } + // Completed records no longer need lifetime bookkeeping. + admission.retain(|f| { + // SAFETY: The vector retains a live reference while queried. + !unsafe { bindings::dma_fence_is_signaled(f.raw()) } + }); + admission.reserve(1, GFP_KERNEL)?; + let mut state = inner.state.lock(); + let queue = state + .queues + .iter() + .find(|q| q.id == data.queue_id) + .ok_or(ENOENT)?; + // Render and compute on even one public queue may complete out of + // order. Give each submission its own timeline: a later independent + // completion must never subsume an earlier dependency in DRM. + let unique = dma_fence::FenceContexts::new(1, c_str!("asahi_m4"), FENCE_KEY)? + .new_fence(0, Completion)?; + let vm_id = queue.vm; + let firmware = queue.firmware.clone(); + let mut timestamps = KVec::new(); + let resolve = |stamp: &uapi::drm_asahi_timestamp| -> Result { + if stamp.handle == 0 { + return if stamp.offset == 0 { + Ok(0) + } else { + Err(EINVAL) + }; + } + let binding = state + .timestamps + .iter() + .find(|t| t.id == stamp.handle) + .ok_or(ENOENT)?; + let offset = u64::from(stamp.offset); + if offset & 7 != 0 || offset.checked_add(8).ok_or(EINVAL)? > binding.size { + return Err(EINVAL); + } + Ok(binding.address + offset) + }; + for command in &commands { + let resolved = match command { + Command::Render(c) => [ + resolve(&c.ts_vtx.start)?, + resolve(&c.ts_vtx.end)?, + resolve(&c.ts_frag.start)?, + resolve(&c.ts_frag.end)?, + ], + Command::Compute(c) => [resolve(&c.ts.start)?, resolve(&c.ts.end)?, 0, 0], + }; + timestamps.push(resolved, GFP_KERNEL)?; + } + let mut timestamp_bindings = KVec::new(); + for binding in &state.timestamps { + timestamp_bindings.push(binding.clone(), GFP_KERNEL)?; + } + let index = state.vms.iter().position(|v| v.id == vm_id).ok_or(ENOENT)?; + let vm = &mut state.vms[index]; + if vm.render_broken { + return Err(EIO); + } + for attachment in &attachments { + vm.cover(attachment.pointer, attachment.size, 4)?; + } + let mut parameters = KVec::new(); + for command in &commands { + parameters.push( + match command { + Command::Render(c) => Parameters::Render(vm.render_parameters(c)?), + Command::Compute(c) => Parameters::Compute(vm.compute_parameters(c)?), + }, + GFP_KERNEL, + )?; + } + let needs_mapping = + (renders != 0 && !vm.render_ready) || (computes != 0 && !vm.compute_ready); + if needs_mapping { + // These reserved namespaces are initialized once before this + // VM's first command on the corresponding engine. + let mut runtime = device.runtime.lock(); + if runtime.render_failed() { + return Err(EIO); + } + if renders != 0 && !vm.render_ready { + vm.render_broken = true; + vm.space + .lock() + .alloc_render_private(vm.kernel.start, vm.kernel.end)?; + let pool = + render::OperandPool::new(vm.kernel.start, vm.kernel.end).ok_or(EINVAL)?; + vm.render_support = Some(Arc::new( + runtime.acquire_render_support(&pool)?, + GFP_KERNEL, + )?); + vm.render_ready = true; + vm.render_broken = false; + } + if computes != 0 && !vm.compute_ready { + vm.render_broken = true; + runtime.map_compute_work(&mut vm.compute_space.lock(), vm.kernel.clone())?; + vm.compute_ready = true; + vm.render_broken = false; + } + } + let mut mappings = KVec::new(); + for mapping in &vm.mappings { + mappings.push(mapping.clone(), GFP_KERNEL)?; + } + let bindings = Bindings { + _mappings: mappings, + _timestamps: timestamp_bindings, + }; + vm.pending.retain(|f| { + // SAFETY: Each entry owns its live fence reference. + !unsafe { bindings::dma_fence_is_signaled(f.raw()) } + }); + vm.pending.reserve(1, GFP_KERNEL)?; + let completion = dma_fence::Fence::from_fence(&unique); + let worker_completion = completion.clone(); + let execution = Arc::pin_init( + try_pin_init!(Execution { + id: NEXT_EXECUTION.fetch_add(1, Ordering::Relaxed), firmware, barriers, + device: device.into(), _owner: inner.state.clone(), + render_space: vm.space.clone(), space: vm.compute_space.clone(), _bindings: bindings, + support: vm.render_support.clone(), + parameters, timestamps, completion: worker_completion, + started: AtomicBool::new(false), work <- new_work!("m4-execution"), + }), + GFP_KERNEL, + )?; + state.vms[index] + .pending + .push(completion.clone(), GFP_KERNEL)?; + let queue = state + .queues + .iter_mut() + .find(|q| q.id == data.queue_id) + .ok_or(ENOENT)?; + let mut job = queue.entity.new_job(1, SubmissionJob { execution })?; + for fence in dependencies { + job.add_dependency(fence)?; + } + // No fallible operation follows output-fence publication. + // Publish the hardware lifetime fence directly. Scheduler finished + // fences can be downgraded to scheduled dependencies by DRM; our + // external inputs must wait for actual hardware completion. + let fence = completion.clone(); + admission.push(completion, GFP_KERNEL)?; + let job = job.arm(); + for output in outputs { + if let Some(chain) = output.chain { + output.object.add_point(chain, &fence, output.point); + } else { + output.object.replace_fence(Some(&fence)); + } + } + job.push(); + Ok(0) + } +} + +impl Vm { + fn cover(&self, start: u64, size: u64, permissions: u32) -> Result { + Mapping::cover(&self.mappings, start, size, permissions) + } + + fn compute_parameters(&self, c: &uapi::drm_asahi_cmd_compute) -> Result { + if c.flags != 0 || c.helper.binary != 0 || c.helper.cfg != 0 || c.helper.data != 0 { + return Err(ENOTSUPP); + } + let size = c + .cdm_ctrl_stream_end + .checked_sub(c.cdm_ctrl_stream_base) + .ok_or(EINVAL)?; + if (c.cdm_ctrl_stream_base | c.cdm_ctrl_stream_end) & 3 != 0 + || size < 4 + || c.sampler_heap & 7 != 0 + || c.sampler_count > 1024 + || (c.sampler_count == 0) != (c.sampler_heap == 0) + { + return Err(EINVAL); + } + self.cover(c.cdm_ctrl_stream_base, size, 2)?; + if c.sampler_count != 0 { + self.cover(c.sampler_heap, u64::from(c.sampler_count) * 8, 2)?; + } + Ok(compute::Parameters { + cdm: c.cdm_ctrl_stream_base, + cdm_end: c.cdm_ctrl_stream_end, + sampler: c.sampler_heap, + sampler_count: c.sampler_count, + scratch: compute::SCRATCH, + marker: compute::MARKER, + save_area: 0, + }) + } + + fn render_parameters(&self, c: &uapi::drm_asahi_cmd_render) -> Result { + use uapi::{ + drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT as DBIAS_IS_INT, + drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as PROCESS_EMPTY_TILES, + }; + + if c.flags & !(PROCESS_EMPTY_TILES | DBIAS_IS_INT) != 0 + || c.vertex_helper.binary != 0 + || c.vertex_helper.cfg != 0 + || c.vertex_helper.data != 0 + || c.fragment_helper.binary != 0 + || c.fragment_helper.cfg != 0 + || c.fragment_helper.data != 0 + { + return Err(ENOTSUPP); + } + let mut p = render::Parameters::default(); + p.set_private(self.kernel.start, self.kernel.end) + .ok_or(EINVAL)?; + p.width = u64::from(c.width_px); + p.height = u64::from(c.height_px); + p.layers = u64::from(c.layers); + p.utile_width = u64::from(c.utile_width_px); + p.utile_height = u64::from(c.utile_height_px); + p.samples = u64::from(c.samples); + p.sample_size = u64::from(c.sample_size_B); + p.tib_blocks = (p.sample_size * p.utile_width * p.utile_height * p.samples).div_ceil(2048); + p.encoder = c.vdm_ctrl_stream_base; + if !p.valid() || p.encoder == 0 || p.encoder & 3 != 0 { + return Err(EINVAL); + } + self.cover(p.encoder, 4, 2)?; + p.utile_config = (p.utile_width / 16) << 12 + | (p.utile_height / 16) << 14 + | match p.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(EINVAL), + }; + p.process_empty_tiles = c.flags & PROCESS_EMPTY_TILES != 0; + p.tile_config = + 0x280 | u64::from(p.layers > 1) | if p.process_empty_tiles { 0x10000 } else { 0 }; + p.multisample_control = c.ppp_multisamplectl; + p.ppp_control = u64::from(c.ppp_ctrl); + p.merge_upper_x_bits = u64::from(c.isp_merge_upper_x); + p.merge_upper_y_bits = u64::from(c.isp_merge_upper_y); + p.aux_fb_flags = 0xc000 | u64::from(c.flags & DBIAS_IS_INT); + p.aux_fb_page_count = 0x100000; + p.scissor_array = c.isp_scissor_base; + p.depth_bias_array = c.isp_dbias_base; + p.occlusion_query_base = c.isp_oclqry_base; + if p.scissor_array == 0 { + return Err(EINVAL); + } + for (address, access) in [ + (p.scissor_array, 2), + (p.depth_bias_array, 2), + (p.occlusion_query_base, 4), + ] { + if address != 0 { + if address & 7 != 0 { + return Err(EINVAL); + } + self.cover(address, 8, access)?; + } + } + for zls in [&c.depth, &c.stencil] { + if (zls.base == 0 && (zls.comp_base != 0 || zls.stride != 0 || zls.comp_stride != 0)) + || (zls.comp_base == 0 && zls.comp_stride != 0) + || (p.layers > 1 && zls.base != 0 && zls.stride == 0) + || (zls.stride != 0 && zls.stride & 0x3fff != 1) + || zls.comp_stride & 0x3fff != 0 + { + return Err(EINVAL); + } + if zls.base != 0 { + let stride = if zls.stride == 0 { + 0 + } else { + (u64::from(zls.stride >> 14) + 1) * 0x4000 + }; + self.cover(zls.base, stride * (p.layers - 1) + 1, 6)?; + } + if zls.comp_base != 0 { + self.cover( + zls.comp_base, + (u64::from(zls.comp_stride >> 14) + 1) * 0x80 * (p.layers - 1) + 1, + 6, + )?; + } + } + p.depth_buffer = c.depth.base; + p.depth_aux_buffer = c.depth.comp_base; + p.depth_stride = u64::from(c.depth.stride); + p.depth_aux_stride = u64::from(c.depth.comp_stride); + p.stencil_buffer = c.stencil.base; + p.stencil_aux_buffer = c.stencil.comp_base; + p.stencil_stride = u64::from(c.stencil.stride); + p.stencil_aux_stride = u64::from(c.stencil.comp_stride); + p.depth_flags = c.zls_ctrl; + p.depth_dimensions = u64::from(c.isp_zls_pixels); + p.depth_clear_value_bits = u64::from(c.isp_bgobjdepth); + p.stencil_clear_value = u64::from(c.isp_bgobjvals); + p.sampler_array = c.sampler_heap; + p.sampler_count = u64::from(c.sampler_count); + if !p.valid() || p.sampler_count > 1024 || p.tib_blocks > 32 { + return Err(EINVAL); + } + if p.sampler_count != 0 { + self.cover(p.sampler_array, p.sampler_count * 8, 2)?; + } + for (program, optional) in [ + (&c.bg, true), + (&c.eot, false), + (&c.partial_bg, c.bg.usc == 0), + (&c.partial_eot, false), + ] { + if program.usc == 0 { + if !optional || program.rsrc_spec != 0 { + return Err(EINVAL); + } + } else { + self.cover((1 << 40) + u64::from(program.usc & !7), 4, 2)?; + } + } + let pipeline = |v: u32| if v == 0 { 0 } else { (1 << 40) + u64::from(v) }; + let load_bind = |v: &uapi::drm_asahi_bg_eot| { + if v.usc == 0 { + 0 + } else { + 0xffff800000000000 | u64::from(v.rsrc_spec) + } + }; + p.load_pipeline = pipeline(c.bg.usc); + p.load_pipeline_bind = load_bind(&c.bg); + p.partial_load_pipeline = pipeline(c.partial_bg.usc); + p.partial_load_pipeline_bind = load_bind(&c.partial_bg); + p.store_pipeline = pipeline(c.eot.usc); + p.store_pipeline_bind = u64::from(c.eot.rsrc_spec); + p.partial_store_pipeline = pipeline(c.partial_eot.usc); + p.partial_store_pipeline_bind = u64::from(c.partial_eot.rsrc_spec); + p.request_tvb_growth = true; + p.tiling_admission = 7; + p.fragment_admission = 7; + p.cycle = 0x240030; + p.tiling_gate = 0xa900400020; + p.fragment_gate = 0xd500400020; + p.tiling_lifecycle = 0x08009f520900a6bb; + p.fragment_lifecycle = 0x08009f520900a6ba; + p.tiling_work_stamp = 0x120; + p.fragment_work_stamp = 0x121; + p.completion_control = 4; + p.layermeta = p.heapmeta; + if p.layers > 1 { + p.heapmeta += 0x100; + } + Ok(p) + } +} + +// SAFETY: These UAPI structures contain only integers and integer arrays. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_cmd_header {} +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_cmd_render {} +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_cmd_compute {} +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_attachment {} + +// SAFETY: VM bind records contain only integer fields. +unsafe impl crate::util::AnyBitPattern for uapi::drm_asahi_gem_bind_op {} diff --git a/drivers/gpu/drm/asahi/g16_fw.rs b/drivers/gpu/drm/asahi/g16_fw.rs new file mode 100644 index 0000000000000000000000000000000000000000..97919511e954ff44763dc8fb4304f9f91b1b3396 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_fw.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from Niklas Sheth's linux-m4-integration prototype. + +//! G16 firmware initialization descriptors, serialized independently of Rust +//! structure alignment. Fields follow the source-built m1n1 G16 model. + +#[path = "g16_queue.rs"] +pub(crate) mod queue; + +pub(crate) const ROOT_ADDRESS: u64 = 0xffff_fc20_0010_0000; +pub(crate) const ROOT_SIZE: usize = 0xc8; +pub(crate) const BUNDLE_ADDRESS: u64 = 0xffff_fc20_c04d_8000; +pub(crate) const BUNDLE_SIZE: usize = 0x20000; +pub(crate) const MAIN_OFFSET: usize = 0x1da40; +pub(crate) const MAIN_SIZE: usize = 0x500; +pub(crate) const CONTROL_DATA: u64 = 0xffff_fc20_c052_8000; +pub(crate) const CONTROL_AUX: u64 = 0xffff_fc20_0014_8000; +pub(crate) const OPERAND_TABLE: u64 = 0x70_0040_8000; +pub(crate) const TIMESTAMP_BASE: u64 = 0xffff_fc21_8140_0000; +pub(crate) const TIMESTAMP_SIZE: u64 = 0x400_0000; + +/// The two publications following firmware's already-consumed slot zero. +pub(crate) fn opening_control() -> [u8; 0x80] { + let mut out = [0u8; 0x80]; + out[..4].copy_from_slice(&0x16u32.to_le_bytes()); + let record = &mut out[0x40..]; + for (offset, value) in [(0, 0x20u32), (4, 1), (8, 0x3f), (0x2c, 20 * 8), (0x34, 1)] { + record[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + put64(record, 0x14, CONTROL_DATA); + put64(record, 0x1c, OPERAND_TABLE); + put64(record, 0x24, OPERAND_TABLE); + out +} + +/// Fresh opening FList; its operand descriptors are admitted by opcode 0x20. +pub(crate) fn opening_flist() -> [u8; 0x80] { + let mut out = [0; 0x80]; + for (off, value) in [(0, 1u32), (0x1c, 0x400000 / 8), (0x28, 1)] { + out[off..off + 4].copy_from_slice(&value.to_le_bytes()); + } + put64(&mut out, 0x14, 0x70_0000_0000); + put64(&mut out, 0x30, OPERAND_TABLE); + put64(&mut out, 0x4c, CONTROL_AUX); + out +} + +/// One freshly owned parameter-buffer descriptor; firmware owns both cursors. +pub(crate) fn parameter_buffer_descriptor(page_list: u64, page_count: u32) -> Option<[u8; 16]> { + if !(0x10_0000_0000..0x20_0000_0000).contains(&page_list) + || page_list & 0x7f != 0 + || page_count == 0 + || page_count >= 1 << 22 + { + return None; + } + let mut out = [0; 16]; + out[..4].copy_from_slice(&((page_list >> 4) as u32 & 0xfffffff8).to_le_bytes()); + out[4..8].copy_from_slice(&page_count.to_le_bytes()); + Some(out) +} + +/// Addresses of 4 KiB accelerator pages within twenty guarded 2 MiB buffers. +pub(crate) fn operand_page(index: usize) -> u64 { + 0x70_0042_0000 + (index / 512) as u64 * 0x208000 + (index % 512) as u64 * 0x1000 +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Three independent cursor addresses followed by the ring address. +pub(crate) struct Channel { + pub(crate) state: [u64; 3], + pub(crate) ring: u64, +} + +pub(crate) struct MainConfig { + pub(crate) hwdata: u64, + pub(crate) repeated: u64, + pub(crate) header: u64, + pub(crate) channels: [Channel; 17], + pub(crate) addresses: [u64; 5], + pub(crate) regions: [u64; 6], + pub(crate) unaligned: u64, +} + +impl MainConfig { + pub(crate) fn bootstrap(bundle: u64) -> Self { + let rings = [ + 0x10240, 0x16240, 0x1c240, 0xea40, 0x14a40, 0x1aa40, 0xd240, 0x13240, 0x19240, 0xba40, + 0x11a40, 0x17a40, + ]; + let states = [ + (0, 0), + (0x40, 0x20), + (0x80, 0x40), + (0x10, 8), + (0x50, 0x28), + (0x90, 0x48), + (0x20, 0x10), + (0x60, 0x30), + (0xa0, 0x50), + (0x30, 0x18), + (0x70, 0x38), + (0xb0, 0x58), + ]; + let channels = core::array::from_fn(|i| { + if i < 12 { + let a = 0xffff_fc20_0002_8000 + states[i].0; + Channel { + state: [a, a + 8, 0xffff_fc20_0002_0000 + states[i].1], + ring: bundle + rings[i], + } + } else { + let (state, ring) = match i { + 12 => ( + [ + 0xffff_fc20_0002_80c0, + 0xffff_fc20_0002_80c8, + 0xffff_fc20_0002_0060, + ], + BUNDLE_ADDRESS + 0x1df00, + ), + 13 => ( + [ + 0xffff_fc20_0003_f1c0, + 0xffff_fc20_0003_f440, + 0xffff_fc20_0003_f200, + ], + 0xffff_fc20_0004_3c40, + ), + 14 => ( + [ + 0xffff_fc20_0003_f3c0, + 0xffff_fc20_000e_5c40, + 0xffff_fc20_0003_f400, + ], + 0xffff_fc20_000e_ec40, + ), + 15 => ([0xffff_fc20_0006_c440, 0, 0], 0), + _ => ([0, 0, 0], 0), + }; + Channel { state, ring } + } + }); + Self { + hwdata: bundle, + repeated: bundle + 0xb980, + header: 0xffff_fc20_0002_0068, + channels, + addresses: [ + 0, + bundle + 0x2680, + bundle + 0x32c0, + bundle + 0x4540, + bundle + 0xb900, + ], + regions: [ + 0xffff_fc20_c051_8000, + 0x70_0027_8000, + 0xffff_fc20_0012_8000, + 0x70_0028_0000, + 0xffff_fc20_0013_0000, + 0, + ], + unaligned: bundle + 0x5380, + } + } + + pub(crate) fn encode(&self) -> [u8; MAIN_SIZE] { + let mut out = [0; MAIN_SIZE]; + for (offset, value) in [ + (0, self.hwdata), + (8, self.repeated), + (16, self.repeated), + (24, self.header), + (0x469, self.unaligned), + ] { + put64(&mut out, offset, value); + } + for (i, channel) in self.channels.iter().enumerate() { + let base = 0x20 + i * 0x20; + for (j, value) in channel.state.iter().enumerate() { + put64(&mut out, base + j * 8, *value); + } + put64(&mut out, base + 24, channel.ring); + } + for (i, value) in self.addresses.iter().enumerate() { + put64(&mut out, 0x24c + i * 8, *value); + } + for (i, value) in self.regions.iter().enumerate() { + put64(&mut out, 0x2c8 + i * 8, *value); + } + for (offset, value) in [(0x2f8, 4u32), (0x3d8, 0xff), (0x4c0, 0x16)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + out + } +} + +pub(crate) struct RootPointers { + pub(crate) region_a: u64, + pub(crate) main_config: u64, + pub(crate) region_c: u64, + pub(crate) status_a: u64, + pub(crate) status_b: u64, + pub(crate) extra_0: u64, + pub(crate) extra_1: u64, +} + +impl RootPointers { + /// Initial firmware namespace. Client addresses are allocated separately. + pub(crate) const fn bootstrap() -> Self { + Self { + region_a: 0xffff_fc20_0010_8000, + main_config: BUNDLE_ADDRESS + MAIN_OFFSET as u64, + region_c: 0xffff_fc20_000f_8000, + status_a: 0xffff_fc20_0003_f180, + status_b: 0xffff_fc20_0003_0000, + extra_0: 0xffff_fc20_0003_eb00, + extra_1: 0xffff_fc20_000f_3440, + } + } + + pub(crate) fn encode(&self) -> [u8; ROOT_SIZE] { + let mut out = [0u8; ROOT_SIZE]; + for (i, value) in [0x04b8u16, 0x8392, 0xc357, 0x0c89].iter().enumerate() { + out[i * 2..i * 2 + 2].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [ + (0x08, self.region_a), + (0x18, self.main_config), + (0x20, self.region_c), + (0xa8, self.status_a), + (0xb0, self.status_b), + (0xb8, self.extra_0), + (0xc0, self.extra_1), + ] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + out[0x2c..0x30].copy_from_slice(&1u32.to_le_bytes()); + out[0x30..0x32].copy_from_slice(&0x4000u16.to_le_bytes()); + out[0x32] = 14; + out[0x33] = 3; + for (i, (shift, entries)) in [(36u8, 64u16), (25, 2048), (14, 2048)].iter().enumerate() { + let level = &mut out[0x34 + i * 0x20..0x54 + i * 0x20]; + level[..4].copy_from_slice(&[8, 14, 14, *shift]); + level[4..6].copy_from_slice(&entries.to_le_bytes()); + level[6..8].copy_from_slice(&0x4000u16.to_le_bytes()); + level[8..16].copy_from_slice(&1u64.to_le_bytes()); + level[16..24].copy_from_slice(&0x0000_03ff_ffff_c000u64.to_le_bytes()); + level[24..32].copy_from_slice(&((u64::from(*entries) - 1) << shift).to_le_bytes()); + } + out + } +} diff --git a/drivers/gpu/drm/asahi/g16_notify.rs b/drivers/gpu/drm/asahi/g16_notify.rs new file mode 100644 index 0000000000000000000000000000000000000000..5cc4e40381b55b4325d967a9e7c20e8c30bc206e --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_notify.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors + +//! RTKit and scheduler wakeups for the single G16 submission worker. + +use kernel::sync::{CondVar, CondVarTimeoutResult, Mutex}; +use kernel::{prelude::*, time::msecs_to_jiffies}; + +#[pin_data] +pub(crate) struct Notifications { + #[pin] + state: Mutex, + #[pin] + changed: CondVar, +} + +struct State { + generation: u64, + crashed: bool, +} + +impl Notifications { + pub(crate) fn new() -> impl PinInit { + pin_init!(Self { + state <- kernel::new_mutex!(State { generation: 0, crashed: false }), + changed <- kernel::new_condvar!(), + }) + } + + /// Called from RTKit's threaded callback or a DRM scheduler callback. + /// Neither callback needs the submission engine or firmware memory lock. + pub(crate) fn notify(&self, crashed: bool) { + let mut state = self.state.lock(); + state.crashed |= crashed; + state.generation = state.generation.wrapping_add(1); + self.changed.notify_one(); + } + + /// Sample before inspecting firmware or ready jobs. A notification between + /// that inspection and sleeping must cause another inspection, not be lost. + pub(crate) fn snapshot(&self) -> Result { + let state = self.state.lock(); + if state.crashed { + return Err(EIO); + } + Ok(state.generation) + } + + /// Sleep until the sampled generation changes or the progress watchdog + /// expires. Spurious wakeups preserve the remaining timeout. + pub(crate) fn wait(&self, generation: u64, timeout_ms: u32) -> Result { + let mut remaining = msecs_to_jiffies(timeout_ms.max(1)); + let mut state = self.state.lock(); + while state.generation == generation && !state.crashed { + remaining = match self + .changed + .wait_interruptible_timeout(&mut state, remaining) + { + CondVarTimeoutResult::Woken { jiffies } + | CondVarTimeoutResult::Signal { jiffies } => jiffies, + CondVarTimeoutResult::Timeout => { + if state.generation == generation && !state.crashed { + return Err(ETIMEDOUT); + } + break; + } + }; + } + if state.crashed { + return Err(EIO); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g16_platform.rs b/drivers/gpu/drm/asahi/g16_platform.rs new file mode 100644 index 0000000000000000000000000000000000000000..a61c7bc60168775466c1e649bcf436fab50fa7dd --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_platform.rs @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors + +//! Live platform inputs and initialization builders from g16g_platform.py and +//! g16g_initdata.py. Firmware data is built from explicit bootloader properties. + +use kernel::{c_str, device, of, prelude::*}; + +const PAGE: u64 = 0x4000; +const REGISTER_BASE: u64 = 0xffff_fc21_8020_0000; +const REGISTER_LIMIT: u64 = 0xffff_fc21_8100_0000; +const MCC_BASE: u64 = 0x2201c4000; +const MCC_SIZE: u32 = 0x18000; + +pub(crate) fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +pub(crate) fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +pub(crate) fn zeroed(size: usize) -> Result> { + let mut out = KVec::new(); + out.extend_with(size, 0, GFP_KERNEL)?; + Ok(out) +} + +/// The boot boundary preserves original ADT bytes, including float bit patterns. +pub(crate) fn adt_word(node: &of::Node, name: &CStr, default: Option) -> Result { + let bytes: Option> = node.get_opt_property(name)?; + match bytes { + Some(bytes) => Ok(u32::from_le_bytes( + bytes.as_slice().try_into().map_err(|_| EINVAL)?, + )), + None => default.ok_or(ENOENT), + } +} + +fn ladder(node: &of::Node, name: &CStr) -> Result<[u32; 11]> { + let values: KVec = node.get_property(name)?; + values.as_slice().try_into().map_err(|_| EINVAL) +} + +pub(crate) struct Register { + pub(crate) slot: usize, + pub(crate) physical: u64, + pub(crate) address: u64, + pub(crate) size: u32, + pub(crate) stride: u32, + pub(crate) relative: u64, + pub(crate) flags: u32, +} + +pub(crate) struct Platform { + pub(crate) node: of::Node, + pub(crate) freq_a: [u32; 11], + pub(crate) freq_b: [u32; 11], + core_voltage: [u32; 11], + memory_voltage: [u32; 11], + index_a: [u32; 11], + index_b: [u32; 11], + relative_a: [u32; 11], + relative_b: [u32; 11], + pub(crate) maximum_power: u32, + pub(crate) core_mask: u32, + revision: u32, + pub(crate) period_ms: u32, + pub(crate) period_clocks: u32, + pub(crate) registers: KVec, + pub(crate) mcc: KVec, + gpc: bool, +} + +impl Platform { + pub(crate) fn new(dev: &device::Device) -> Result { + let node = dev.of_node().ok_or(ENODEV)?; + // Only the maximum is consumed by the firmware builders. A loader + // with an observed board maximum need not invent an entire power + // ladder. Continue accepting the original loader's modeled ladder. + let maximum_power = match node.get_opt_property::(c_str!("apple,m4-max-power-mw"))? { + Some(value) => value, + None => { + let power = ladder(&node, c_str!("apple,m4-power-mw"))?; + if power[0] != 0 || power.windows(2).any(|w| w[0] > w[1]) { + return Err(EINVAL); + } + power[10] + } + }; + let freq_a = ladder(&node, c_str!("apple,m4-freq-a"))?; + let freq_b = ladder(&node, c_str!("apple,m4-freq-b"))?; + if maximum_power == 0 + || maximum_power > i32::MAX as u32 + || freq_a[0] != 0 + || freq_a.windows(2).any(|w| w[0] >= w[1]) + || freq_b[0] != 0 + || freq_b.windows(2).any(|w| w[0] >= w[1]) + { + return Err(EINVAL); + } + let period_clocks = match adt_word( + &node, + c_str!("apple,sgx-gpu-pwr-sample-period-aic-clks"), + None, + ) { + Ok(value) => value, + Err(ENOENT) => adt_word(&node, c_str!("apple,sgx-gpu-power-sample-period"), None)? + .checked_mul(24000) + .ok_or(EINVAL)?, + Err(error) => return Err(error), + }; + let period_ms = period_clocks / 24000; + if period_ms == 0 { + return Err(EINVAL); + } + let core_mask: u32 = node.get_property(c_str!("apple,m4-core-mask"))?; + if core_mask == 0 || core_mask & !0x3ff != 0 { + return Err(EINVAL); + } + let mut this = Self { + core_voltage: ladder(&node, c_str!("apple,m4-core-voltage"))?, + memory_voltage: ladder(&node, c_str!("apple,m4-memory-voltage"))?, + index_a: ladder(&node, c_str!("apple,m4-index-a"))?, + index_b: ladder(&node, c_str!("apple,m4-index-b"))?, + relative_a: ladder(&node, c_str!("apple,m4-relative-a"))?, + relative_b: ladder(&node, c_str!("apple,m4-relative-b"))?, + revision: node.get_property(c_str!("apple,chip-revision"))?, + gpc: adt_word(&node, c_str!("apple,sgx-no-gpc"), Some(0))? == 0, + node, + freq_a, + freq_b, + maximum_power, + core_mask, + period_ms, + period_clocks, + registers: KVec::new(), + mcc: KVec::new(), + }; + this.build_registers()?; + Ok(this) + } + + fn build_registers(&mut self) -> Result { + let reg: KVec = self.node.get_property(c_str!("reg"))?; + if reg.len() != 8 { + return Err(EINVAL); + } + let base = (u64::from(reg[4]) << 32) | u64::from(reg[5]); + let span = (u64::from(reg[6]) << 32) | u64::from(reg[7]); + let irq: KVec = self + .node + .get_property(c_str!("apple,sgx-meta-sw-interrupt"))?; + if irq.len() != 20 && irq.len() != 40 { + return Err(EINVAL); + } + let irq_address = u64::from_le_bytes( + irq[irq.len() - 20..irq.len() - 12] + .try_into() + .map_err(|_| EINVAL)?, + ); + if irq_address < PAGE || irq_address >= 1 << 48 { + return Err(EINVAL); + } + let mask = adt_word( + &self.node, + c_str!("apple,sgx-active-mcc-bit-vector"), + Some(3), + )?; + if mask == 0 { + return Err(EINVAL); + } + for bit in 0..32 { + if mask & (1 << bit) != 0 { + self.mcc.push(MCC_BASE + (bit << 25), GFP_KERNEL)?; + } + } + self.registers.push( + Register { + slot: 0, + physical: irq_address & !(PAGE - 1), + address: 0, + size: PAGE as u32, + stride: PAGE as u32, + relative: 0, + flags: 2, + }, + GFP_KERNEL, + )?; + self.registers.push( + Register { + slot: 3, + physical: self.mcc[0], + address: 0, + size: self.mcc.len() as u32 * MCC_SIZE, + stride: MCC_SIZE, + relative: 0, + flags: 2, + }, + GFP_KERNEL, + )?; + // Native HAL declarations and the three platform-specific windows. + for (slot, offset, size, relative) in [ + (17, 0, 0x20000, true), + (26, 0xd04000, 0x8000, true), + (27, 0xd0d000, 0x1000, true), + (28, 0xd58000, 0x8000, true), + (29, 0xd10000, 0x4000, true), + (31, 0xd40000, 0x4000, true), + (32, 0xd60000, 0x8000, true), + (40, 0xe1c000, 0x4000, true), + (22, 0x1000000, 0x8000, false), + (39, 0xe08000, 0x8000, false), + (41, 0xe1f800, 0x4000, false), + ] { + if offset + u64::from(size) > span { + return Err(EINVAL); + } + self.registers.push( + Register { + slot, + physical: base.checked_add(offset).ok_or(EINVAL)?, + address: 0, + size, + stride: size, + relative: if relative { offset } else { 0 }, + flags: 2, + }, + GFP_KERNEL, + )?; + } + for (slot, physical, size, flags) in [ + (9, 0x3803d0000, 0x1000, 2), + (10, 0x3803c0000, 0x2000, 0), + (12, 0x50165c000, 0x4000, 2), + (14, 0x380280000, 0x8000, 0), + (15, 0x38c840000, 0x24000, 0), + ] { + self.registers.push( + Register { + slot, + physical, + address: 0, + size, + stride: size, + relative: 0, + flags, + }, + GFP_KERNEL, + )?; + } + self.registers.sort_unstable_by_key(|r| r.slot); + let mut cursor = REGISTER_BASE; + for r in &mut self.registers { + let offset = r.physical & (PAGE - 1); + r.address = cursor + offset; + cursor += (offset + u64::from(r.size) + PAGE - 1) & !(PAGE - 1); + cursor += PAGE; + if cursor > REGISTER_LIMIT { + return Err(ENOSPC); + } + } + Ok(()) + } + + pub(crate) fn hwdata(&self) -> Result> { + let mut out = zeroed(0x3db4)?; + for r in &self.registers { + let off = 0x640 + r.slot * 0x28; + put64(&mut out, off, r.physical); + put64(&mut out, off + 8, r.address); + put32(&mut out, off + 0x10, r.size); + put32(&mut out, off + 0x14, r.stride); + put64(&mut out, off + 0x18, r.relative); + put32(&mut out, off + 0x20, r.flags); + } + for (base, repeats, frequencies) in [(0xfc8, 16, &self.freq_a), (0x1cdc, 1, &self.freq_b)] { + for state in 0..11 { + put32(&mut out, base + state * 4, frequencies[state]); + for word in 0..repeats { + put32( + &mut out, + base + 0x40 + state * 0x40 + word * 4, + self.core_voltage[state], + ); + put32( + &mut out, + base + 0x440 + state * 0x40 + word * 4, + self.memory_voltage[state], + ); + } + } + } + for (off, values) in [ + (0x1808, &self.freq_b), + (0x18c8, &self.relative_a), + (0x1908, &self.relative_b), + (0x19c8, &self.index_a), + (0x1a08, &self.index_b), + ] { + for (i, value) in values.iter().enumerate() { + put32(&mut out, off + i * 4, *value); + } + } + for i in 0..11 { + put32(&mut out, 0x1848 + i * 4, 1.02f32.to_bits()); + } + for (off, value) in [ + (0xe90, 0x8132), + (0xe94, self.revision >> 4), + (0xe98, self.revision & 7), + (0xea0, u32::from(self.gpc)), + (0xeb8, 1), + (0xec8, 1), + (0xed0, 24000), + (0xed8, self.period_ms), + (0xee0, 1), + (0xee4, 1), + (0xee8, 1), + (0xfc4, 10), + (0x1cd8, 10), + (0x2604, 1), + ] { + put32(&mut out, off, value); + } + out[0x258e..0x2590].copy_from_slice(&22u16.to_le_bytes()); + for group in 0..2 { + for word in 1..4 { + put32(&mut out, 0x2590 + group * 16 + word * 4, 1); + } + } + for word in 0..12 { + put32(&mut out, 0x25b4 + word * 4, u32::MAX); + } + Ok(out) + } + + pub(crate) fn region_c(&self) -> Result> { + let mut out = zeroed(PAGE as usize)?; + for (off, value) in [ + (0x78, 1), + (0x7ec, 1), + (0x7f0, 1), + (0xe28, 3), + (0xe2c, 1), + (0x24, 4096), + (0x998, 32), + (0x99c, 8), + (0x9a0, 256), + (0x9a4, 1), + (0x9c8, 64), + (0x9cc, 64), + (0x9d0, 2), + (0x84, self.maximum_power), + (0x88, 1000), + (0x8c, 1000), + (0x98, 1000), + ] { + put32(&mut out, off, value); + } + for (off, value) in [(0x54, u16::MAX), (0x56, 40), (0x58, u16::MAX)] { + out[off..off + 2].copy_from_slice(&value.to_le_bytes()); + } + Ok(out) + } +} diff --git a/drivers/gpu/drm/asahi/g16_power.rs b/drivers/gpu/drm/asahi/g16_power.rs new file mode 100644 index 0000000000000000000000000000000000000000..2ffab79f746fe685071e4f98c70843be67b9d446 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_power.rs @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors + +//! Power configuration from g16g_platform.py and g16g_initdata.py. +//! ADT values are original platform inputs; every firmware field is built here. + +use crate::{ + f32, + float::F32, + g16_platform::{adt_word, put32, put64, zeroed, Platform}, +}; +use kernel::{c_str, prelude::*}; + +fn putf(out: &mut [u8], offset: usize, value: F32) -> Result { + if value.to_bits() & 0x7f80_0000 == 0x7f80_0000 { + return Err(EINVAL); + } + put32(out, offset, value.to_bits()); + Ok(()) +} + +fn coefficients(count: u32) -> Result<(F32, F32)> { + if count == 0 { + return Err(EINVAL); + } + let count = F32::from(count); + Ok(((count - f32!(1.0)) / count, f32!(1.0) / count)) +} + +struct Filter { + alpha: F32, + inverse: F32, + count_x4: u32, + duration: u32, + clocks: u64, +} + +fn filter(count: u32, period: u32, request: Option) -> Result { + let (alpha, inverse) = coefficients(count)?; + let count_x4 = count.checked_mul(4).ok_or(EINVAL)?; + let duration = count.checked_mul(period).ok_or(EINVAL)?; + let clocks = u64::from(duration.checked_mul(24000).ok_or(EINVAL)?); + if let Some(ms) = request { + if period == 0 || ms / period != count { + return Err(EINVAL); + } + } + Ok(Filter { + alpha, + inverse, + count_x4, + duration: request.unwrap_or(duration), + clocks, + }) +} + +impl Platform { + /// Entire source-built bundle, including the firmware's power controllers. + pub(crate) fn bundle(&self, entropy: u32) -> Result> { + let mut out = zeroed(crate::g16_fw::BUNDLE_SIZE)?; + let hwdata = self.hwdata()?; + out[..hwdata.len()].copy_from_slice(&hwdata); + let main = crate::g16_fw::MainConfig::bootstrap(crate::g16_fw::BUNDLE_ADDRESS).encode(); + out[crate::g16_fw::MAIN_OFFSET..crate::g16_fw::MAIN_OFFSET + main.len()] + .copy_from_slice(&main); + put64(&mut out, 0x8ed0, 0xffff_fc20_000f_3440 + 0x80); + put64(&mut out, 0xb698, 0xffff_fc20_c050_0000); + self.power_data(&mut out)?; + put32(&mut out, 0xb754, entropy); + Ok(out) + } + + pub(crate) fn power_data(&self, out: &mut [u8]) -> Result { + if out.len() < 0xb758 || self.period_ms == 0 { + return Err(EINVAL); + } + // The property spellings and defaults follow the Python reader exactly. + macro_rules! uint { + ($name:literal, $default:expr) => { + adt_word( + &self.node, + c_str!(concat!("apple,sgx-", $name)), + Some($default), + )? + }; + } + macro_rules! optional { + ($name:literal) => { + match adt_word(&self.node, c_str!(concat!("apple,sgx-", $name)), None) { + Ok(value) => Some(value), + Err(ENOENT) => None, + Err(e) => return Err(e), + } + }; + } + macro_rules! gain { + ($name:literal, $default:expr) => { + F32::from_bits(uint!($name, f32!($default).to_bits())) + }; + } + macro_rules! ints { + ($($offset:expr => $value:expr),* $(,)?) => { + $(put32(out, $offset, $value);)* + }; + } + macro_rules! floats { + ($($offset:expr => $value:expr),* $(,)?) => { + $(putf(out, $offset, $value)?;)* + }; + } + let max_scaled = 100 * (self.freq_a.len() as u32 - 1); + let maximum = F32::from(self.maximum_power); + let duty = uint!("gpu-pwr-min-duty-cycle", 40); + let minimum = uint!("gpu-pwr-integral-min-clamp", 0); + if duty > max_scaled || minimum > self.maximum_power { + return Err(EINVAL); + } + let minimum = F32::from(minimum); + let dt = F32::from(self.period_ms) / f32!(1000.0); + let micros = self.period_ms.checked_mul(1000).ok_or(EINVAL)?; + // Sixteen rows of signed 16-bit initial power slots. + for i in 0..16 * 32 { + out[0x6f6f + i * 2..0x6f71 + i * 2].copy_from_slice(&i16::MAX.to_le_bytes()); + } + ints!(0x5384 => self.period_clocks, 0x5388 => self.period_clocks, 0x5d38 => 625); + for i in 0..11 { + put32(out, 0x5440 + i * 4, f32!(1.02).to_bits()); + } + let (alpha, inverse) = coefficients(uint!("gpu-pwr-filter-time-constant", 313))?; + floats!(0x5d44 => alpha, 0x5d4c => inverse, + 0x5d54 => gain!("gpu-pwr-integral-gain", 0.0202129), + 0x5d5c => minimum, 0x5d60 => maximum, + 0x5d64 => gain!("gpu-pwr-proportional-gain", 5.2831855), + 0x5d6c => -F32::from(max_scaled) / maximum); + ints!(0x5d70 => duty, 0x5d74 => max_scaled, 0x5d78 => max_scaled, 0x5db8 => max_scaled); + for off in [0x5d84, 0x7c48, 0x7c4c, 0x7c50] { + put32(out, off, self.maximum_power); + } + floats!(0x7fb0 => f32!(65536.0), 0x7fb4 => f32!(0.0)); + ints!(0x7fc0 => duty, 0x7fc4 => max_scaled); + floats!(0x7570 => f32!(65536.0), 0x7574 => F32::from(duty), 0x7578 => F32::from(max_scaled), + 0x7598 => f32!(100.0)); + ints!(0x757c => self.freq_a[10], 0x7590 => duty, 0x759c => max_scaled, 0x7bf0 => max_scaled); + let (alpha, inverse) = coefficients(5)?; + floats!(0x75a4 => alpha, 0x75a8 => inverse); + + let ppm_ms = uint!("gpu-ppm-filter-time-constant-ms", 16); + let ppm = filter(ppm_ms / self.period_ms, self.period_ms, Some(ppm_ms))?; + floats!(0x5dfc => ppm.alpha, 0x5e04 => ppm.inverse, + 0x5e0c => dt * gain!("gpu-ppm-ki", 150.0), + 0x5e14 => minimum, 0x5e18 => f32!(65536.0), 0x5e1c => gain!("gpu-ppm-kp", 0.25)); + ints!(0x5df0 => ppm.count_x4, 0x5e28 => duty, 0x5e2c => max_scaled, + 0x5e3c => self.maximum_power, 0x5e48 => ppm.duration); + put64(out, 0x5e50, ppm.clocks); + + let base = uint!("gpu-perf-base-pstate", 1); + let utilization = uint!("gpu-perf-tgt-utilization", 85); + let dead_zone = uint!("gpu-perf-dz", 0); + let boost = uint!("gpu-perf-boost-min-util", 95); + let integral_minimum = uint!("gpu-perf-integral-min-clamp", 0); + if base == 0 + || base >= 11 + || utilization > 100 + || dead_zone > utilization + || boost > 100 + || integral_minimum > 95 + { + return Err(EINVAL); + } + let base_scaled = 100 * base; + let target = utilization - dead_zone; + for off in [0x5390, 0x53e0] { + put32(out, off, 4); + putf(out, off + 4, f32!(1.0))?; + } + for off in [0x53ac, 0x53b0, 0x53ec, 0x53f0] { + put32(out, off, 1); + } + for off in [0x53d0, 0x5410] { + put32(out, off, 100); + } + for off in [0x53bc, 0x53fc] { + for (i, value) in [0, base_scaled, 1, max_scaled].iter().enumerate() { + put32(out, off + i * 4, *value); + } + } + let first = coefficients(uint!("gpu-perf-filter-time-constant", 5))?; + let second = coefficients(uint!("gpu-perf-filter-time-constant2", 200))?; + floats!(0x5ecc => first.0, 0x5ed0 => second.0, 0x5ed4 => first.1, 0x5ed8 => second.1, + 0x5edc => gain!("gpu-perf-integral-gain", 0.7975), + 0x5ee0 => gain!("gpu-perf-integral-gain2", 0.7975), + 0x5ee4 => F32::from(integral_minimum), 0x5ee8 => f32!(95.0), + 0x5eec => gain!("gpu-perf-proportional-gain", 5.4), + 0x5ef0 => gain!("gpu-perf-proportional-gain2", 5.4), + 0x5ef4 => F32::from(max_scaled-base_scaled) / f32!(95.0)); + ints!(0x5ea8 => target, 0x5eb0 => boost, + 0x5eb4 => uint!("gpu-perf-boost-ce-step", 50), + 0x5eb8 => uint!("gpu-perf-reset-iters", 6), 0x5ec0 => 6, 0x5ec4 => 1, + 0x5ec8 => uint!("gpu-perf-filter-drop-threshold", 0), + 0x5ef8 => base_scaled, 0x5efc => max_scaled, 0x5f00 => base_scaled, + 0x5f0c => target, 0x5f14 => dead_zone, 0x5f40 => base_scaled); + + let first = filter( + uint!("gpu-se-filter-time-constant", 9), + self.period_ms, + None, + )?; + let second = filter( + uint!("gpu-se-filter-time-constant-1", 3), + self.period_ms, + None, + )?; + floats!(0x7ed8 => first.alpha, 0x7edc => second.alpha, + 0x7ee0 => first.inverse, 0x7ee4 => second.inverse, + 0x7ee8 => gain!("gpu-se-ki", -50.0) * dt, + 0x7eec => gain!("gpu-se-ki-1", -100.0) * dt, + 0x7ef0 => f32!(0.0), 0x7ef4 => f32!(65536.0), + 0x7ef8 => gain!("gpu-se-kp", -5.0), 0x7efc => gain!("gpu-se-kp-1", -5.0), + 0x7f40 => f32!(65536.0), 0x7f14 => F32::from(micros)); + ints!(0x7f04 => 100, 0x7f08 => max_scaled, 0x7f0c => base_scaled, + 0x7f24 => first.duration, 0x7f28 => second.duration, 0x7f68 => micros, + 0x7ecc => 50, 0x7ed0 => 1, 0x7f18 => 1400, + 0x7f6c => (33+self.period_ms/2)/self.period_ms, + 0x7f70 => uint!("gpu-se-inactive-threshold", 2500), + 0x7f74 => uint!("gpu-se-engagement-criteria", 600), + 0x7f78 => uint!("gpu-se-engagement-tolerance", 2), + 0x7f7c => uint!("gpu-se-reset-tolerance", 4), + 0x7f80 => uint!("gpu-se-reset-criteria", 50)); + put64(out, 0x7f2c, first.clocks); + put64(out, 0x7f34, second.clocks); + + let average_target = filter( + uint!("gpu-avg-power-target-filter-tc", 1), + self.period_ms, + None, + )?; + floats!(0x7c5c => average_target.alpha, 0x7c60 => average_target.inverse); + ints!(0x7c64 => average_target.count_x4, 0x7c68 => average_target.duration); + put64(out, 0x7c6c, average_target.clocks); + let ki_only = optional!("gpu-avg-power-ki-only"); + let direct_ki = optional!("gpu-avg-power-ki"); + let mut avg_ki = F32::from_bits(ki_only.or(direct_ki).unwrap_or(f32!(16.875).to_bits())); + if ki_only.is_some() || direct_ki.is_none() { + avg_ki = avg_ki * dt; + } + let avg_duty = uint!("gpu-avg-power-min-duty-cycle", 40); + if avg_duty > max_scaled { + return Err(EINVAL); + } + floats!(0x7d58 => avg_ki, 0x7d60 => f32!(0.0), 0x7d64 => f32!(65536.0), + 0x7d68 => gain!("gpu-avg-power-kp", 2.42), 0x7d70 => f32!(0.0), 0x7d84 => maximum); + ints!(0x7d74 => avg_duty, 0x7d78 => max_scaled, 0x7d7c => max_scaled, + 0x7d88 => self.maximum_power, 0x7dbc => max_scaled); + let request = optional!("gpu-avg-power-filter-tc-ms"); + let count = match request { + Some(ms) => ms / self.period_ms, + None => uint!("gpu-avg-power-input-filter-tc", 248 / self.period_ms), + }; + let average = filter(count, self.period_ms, request)?; + ints!(0x7d3c => average.count_x4, 0x7d94 => average.duration); + floats!(0x7d48 => average.alpha, 0x7d50 => average.inverse); + put64(out, 0x7d9c, average.clocks); + + // Fast-die uses the untruncated AIC interval, unlike the other loops. + let fast_dt = F32::from(self.period_clocks) / f32!(24000.0) / f32!(1000.0); + let release = uint!("gpu-fast-die0-release-temp", 80); + let delta = uint!("gpu-fast-die0-prop-tgt-delta", 0); + if release > u32::MAX / 100 || delta > u32::MAX / 100 { + return Err(EINVAL); + } + put64( + out, + 0x75e8, + (1 << 3) | (1 << 4) | (1 << 6) | (1 << 8) | (1 << 11) | (1 << 13) | (1 << 14), + ); + ints!(0x75f0 => (F32::from(release)*f32!(100.0)).to_u32_checked().ok_or(EINVAL)?, + 0x75f8 => 4, 0x7630 => duty, 0x7634 => max_scaled, 0x7638 => max_scaled, + 0x763c => (F32::from(delta)*f32!(100.0)).to_u32_checked().ok_or(EINVAL)?, + 0x7644 => 11000, 0x7678 => max_scaled); + floats!(0x7604 => f32!(0.0), 0x760c => f32!(1.0), + 0x7614 => gain!("gpu-fast-die0-integral-gain", 200.0)*fast_dt, + 0x761c => f32!(0.0), 0x7620 => f32!(65536.0), + 0x7624 => gain!("gpu-fast-die0-proportional-gain", 120.0), 0x762c => f32!(0.0)); + + let turbo_input = optional!("gpu-turbo-controller-input"); + let mode = turbo_input.unwrap_or(2); + let (kp, ki, target, delta, threshold) = if mode == 0 { + (f32!(300.0), f32!(1.3), 9600, 300, 0) + } else { + ( + f32!(250.0), + f32!(1.15), + if mode == 1 { 0 } else { 10150 }, + 0, + 700, + ) + }; + let (alpha, inverse) = coefficients(200)?; + out[0x7898] = u8::from(turbo_input.is_some()); + out[0x7899] = u8::from(mode == 1); + floats!(0x7aaa => alpha, 0x7ab2 => inverse, 0x7aba => ki, 0x7ac2 => f32!(0.0), + 0x7ac6 => f32!(65536.0), 0x7aca => kp, 0x7ad2 => f32!(-100.0/65536.0), + 0x7ae6 => f32!(3000.0), 0x7b56 => f32!(64.0), 0x7b5a => f32!(65536.0)); + ints!(0x7ad6 => 900, 0x7ada => 1000, 0x7ade => 1000, 0x7aea => target, + 0x7af2 => delta, 0x7b1e => 1000, 0x7b3a => 200, 0x7b52 => threshold); + + let (alpha, inverse) = coefficients(32)?; + floats!(0x81b8 => alpha, 0x81c0 => inverse, 0x81c8 => f32!(5.0), + 0x81d0 => f32!(0.0), 0x81d4 => f32!(65536.0), 0x81d8 => f32!(15.0)); + ints!(0x81ac => 4, 0x81e4 => max_scaled-100); + for off in [0x81e8, 0x81ec, 0x81f8, 0x822c] { + put32(out, off, max_scaled); + } + for (off, key) in [ + (0xb6a0, *b"g0TR"), + (0xb708, *b"g0t0"), + (0xb70c, *b"g0t1"), + (0xb710, *b"g0EB"), + (0xb714, *b"g0EL"), + ] { + put32(out, off, u32::from_be_bytes(key)); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g16_queue.rs b/drivers/gpu/drm/asahi/g16_queue.rs new file mode 100644 index 0000000000000000000000000000000000000000..e270a49adfd711f582d054e1421280360a6ffe47 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_queue.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from Niklas Sheth's linux-m4-integration prototype. + +//! G16 queue transport fields, independent of allocation addresses. + +fn u32_at(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn u64_at(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +pub(crate) struct Queue { + pub(crate) pointers: u64, + pub(crate) ring: u64, + pub(crate) jobs: u64, + pub(crate) private: u64, + pub(crate) context: u64, + pub(crate) uuid: u32, +} + +impl Queue { + pub(crate) fn encode(&self) -> [u8; 0xb0] { + let mut out = [0; 0xb0]; + for (off, value) in [ + (0, self.pointers), + (8, self.ring), + (0x10, self.jobs), + (0x18, self.private), + (0xa4, self.context), + ] { + u64_at(&mut out, off, value); + } + // Priority zero's scheduler profile. + u64_at(&mut out, 0x38, 0xffff_ffff_ffff_0000); + for (off, value) in [ + (0x2c, u32::MAX), + (0x40, 1), + (0x48, 1), + (0x4c, u32::MAX), + (0x50, self.uuid), + ] { + u32_at(&mut out, off, value); + } + out + } +} + +impl Queue { + pub(crate) fn encode_priority(&self, priority: u32) -> Option<[u8; 0xb0]> { + let (mask, first, last) = match priority { + 0 => (0xffff_ffff_ffff_0000, 1, 1), + 1 => (0xffff_ffff_0000_0000, 0, 0), + 2 => (0xffff_0000_0000_0000, 0, 2), + 3 => (0, 0, 3), + _ => return None, + }; + let mut out = self.encode(); + u32_at(&mut out, 0x30, priority); + u32_at(&mut out, 0x34, priority); + u64_at(&mut out, 0x38, mask); + u32_at(&mut out, 0x40, first); + u32_at(&mut out, 0x48, last); + Some(out) + } +} + +pub(crate) fn pointers(capacity: u32, write: u32) -> [u8; 0x60] { + let mut out = [0; 0x60]; + u32_at(&mut out, 0x40, write); + u32_at(&mut out, 0x50, capacity); + out +} + +pub(crate) fn context() -> [u8; 0x40] { + let mut out = [0; 0x40]; + out[..2].copy_from_slice(&[0xff, 0xff]); + out[5] = 1; + out[0x26] = 2; + out[0x33] = 0xff; + out +} + +pub(crate) fn jobs(address: u64) -> [u8; 0x18] { + let mut out = [0; 0x18]; + u64_at(&mut out, 8, address); + out +} + +pub(crate) fn barrier(stamp: u64, wait: u32, event: u32, stamp_self: u32, uuid: u32) -> [u8; 0x40] { + let mut out = [0; 0x40]; + u32_at(&mut out, 0, 4); + u64_at(&mut out, 4, stamp); + for (off, value) in [(0xc, wait), (0x10, event), (0x14, stamp_self), (0x18, uuid)] { + u32_at(&mut out, off, value); + } + out +} + +/// General G16 dependency, before a consumer TA or CDM Work. Type 1 participates +/// in firmware's dynamic dependency graph; the internal TA-to-fragment prelude +/// retains type 0 for partial-render scheduling. +pub(crate) fn dependency( + stamp: u64, + wait: u32, + event: u32, + stamp_self: u32, + uuid: u32, +) -> [u8; 0x40] { + let mut out = [0; 0x40]; + u32_at(&mut out, 0, 4); + u64_at(&mut out, 4, stamp); + u64_at(&mut out, 0xc, stamp); + for (off, value) in [ + (0x14, wait), + (0x20, event), + (0x24, stamp_self), + (0x28, uuid), + (0x30, 1), + ] { + u32_at(&mut out, off, value); + } + out +} + +pub(crate) fn channel( + queue: u64, + head: u16, + event: u8, + kind: u32, + first: bool, + tag: u64, +) -> [u8; 0x18] { + let mut out = [0; 0x18]; + u64_at(&mut out, 0, tag); + u64_at(&mut out, 8, queue); + u32_at(&mut out, 0x10, kind); + u32_at( + &mut out, + 0x14, + u32::from(head) | (u32::from(event) << 16) | (u32::from(first) << 24), + ); + out +} + +// Bootstrap completion stamp; public workqueues allocate their own storage. +pub(crate) const STAMP: u64 = 0xffff_fc20_0013_8000; diff --git a/drivers/gpu/drm/asahi/g16_render.rs b/drivers/gpu/drm/asahi/g16_render.rs new file mode 100644 index 0000000000000000000000000000000000000000..c913be4924cef4b511324cd544f5f08c1a32df14 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_render.rs @@ -0,0 +1,1167 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from Niklas Sheth's linux-m4-integration prototype. + +//! T8132 render register ABI. Ordered writes are significant, including duplicates. +//! Ported from the open m1n1 G16/G17 register model. Addresses and lifecycle +//! values are supplied by the runtime; geometry is derived from each render. + +#[derive(Clone, Copy, Default)] +pub(crate) struct Parameters { + pub(crate) request_tvb_growth: bool, + pub(crate) fragment_sync_grow: bool, + pub(crate) process_empty_tiles: bool, + pub(crate) partial_load_pipeline: u64, + pub(crate) partial_load_pipeline_bind: u64, + pub(crate) partial_store_pipeline: u64, + pub(crate) partial_store_pipeline_bind: u64, + pub(crate) sample_size: u64, + pub(crate) sampler_array: u64, + pub(crate) sampler_count: u64, + + pub(crate) aux_fb: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_page_count: u64, + pub(crate) completion_control: u64, + pub(crate) context_base: u64, + pub(crate) cycle: u64, + pub(crate) deflake_1: u64, + pub(crate) deflake_2: u64, + pub(crate) deflake_3: u64, + pub(crate) depth_aux_buffer: u64, + pub(crate) depth_aux_stride: u64, + pub(crate) depth_bias_array: u64, + pub(crate) depth_buffer: u64, + pub(crate) depth_clear_value_bits: u64, + pub(crate) depth_dimensions: u64, + pub(crate) depth_flags: u64, + pub(crate) depth_stride: u64, + pub(crate) encoder: u64, + pub(crate) fragment_admission: u64, + pub(crate) fragment_gate: u64, + pub(crate) fragment_lifecycle: u64, + pub(crate) fragment_status: u64, + pub(crate) fragment_work_stamp: u64, + pub(crate) heapmeta: u64, + pub(crate) height: u64, + pub(crate) layermeta: u64, + pub(crate) layers: u64, + pub(crate) load_pipeline: u64, + pub(crate) load_pipeline_bind: u64, + pub(crate) merge_upper_x_bits: u64, + pub(crate) merge_upper_y_bits: u64, + pub(crate) multisample_control: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) ppp_control: u64, + pub(crate) scene_slot: u64, + pub(crate) samples: u64, + pub(crate) scissor_array: u64, + pub(crate) stencil_aux_buffer: u64, + pub(crate) stencil_aux_stride: u64, + pub(crate) stencil_buffer: u64, + pub(crate) stencil_clear_value: u64, + pub(crate) stencil_stride: u64, + pub(crate) store_pipeline: u64, + pub(crate) store_pipeline_bind: u64, + pub(crate) ta_status: u64, + pub(crate) tib_blocks: u64, + pub(crate) tile_config: u64, + pub(crate) tilemap: u64, + pub(crate) tiling_admission: u64, + pub(crate) tiling_gate: u64, + pub(crate) tiling_lifecycle: u64, + pub(crate) tiling_work_stamp: u64, + pub(crate) tpc: u64, + pub(crate) utile_config: u64, + pub(crate) utile_height: u64, + pub(crate) utile_width: u64, + pub(crate) width: u64, +} + +struct Geometry { + tiles_x: u64, + tiles_y: u64, + size1: u64, + size2: u64, + size3: u64, + x_blocks: u64, + y_blocks: u64, + screen: u64, + pixels: u64, + macro_size: u64, +} + +impl Parameters { + pub(crate) fn valid(&self) -> bool { + (1..=16384).contains(&self.width) + && (1..=16384).contains(&self.height) + && (1..=2048).contains(&self.layers) + && matches!( + (self.utile_width, self.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) + && matches!(self.samples, 1 | 2 | 4) + && self.tib_blocks <= 64 + && self.scene_slot < 36 + && (self.sampler_array == 0) == (self.sampler_count == 0) + && self.sampler_array & 7 == 0 + && self.sampler_count < u32::MAX as u64 + && [ + self.tilemap, + self.heapmeta, + self.layer_metadata(), + self.tpc, + self.deflake_1, + self.deflake_2, + self.deflake_3, + self.encoder, + ] + .iter() + .all(|a| { + a.checked_sub(self.context_base) + .is_some_and(|v| v <= u32::MAX as u64) + }) + } + + fn ta_offset(&self, address: u64) -> u64 { + address - self.context_base + } + fn layer_metadata(&self) -> u64 { + if self.layermeta == 0 { + self.heapmeta + } else { + self.layermeta + } + } + fn tib_encoding(&self) -> u64 { + // T8132's 1..8 RGBA16F attachment sweep covers the full 128 KiB + // allocation. The low field wraps at 64 blocks; multisample renders + // double their bank tag above 64 KiB. Bit 6 is not an allocation bit. + let blocks = self.tib_blocks & 0x3f; + if self.samples == 1 { + blocks + } else { + let banks = if self.tib_blocks > 32 { 2 } else { 1 }; + ((self.samples * banks) << 16) | blocks + } + } + fn geometry(&self) -> Geometry { + let tx = self.width.div_ceil(32); + let ty = self.height.div_ceil(32); + let mx = (tx.div_ceil(4) + 3) & !3; + let my = (ty.div_ceil(4) + 3) & !3; + let ux = 32 / self.utile_width; + let uy = 32 / self.utile_height; + Geometry { + tiles_x: tx, + tiles_y: ty, + size1: (5 * mx * my * ux * uy).div_ceil(4), + size2: mx * my, + size3: 2 * mx * my * ux * uy, + x_blocks: 3 * mx | ((2 * mx) << 9) | (mx << 18), + y_blocks: 3 * my | ((2 * my) << 9) | (my << 18), + screen: ((ty - 1) << 12) | (tx - 1), + pixels: (self.width - 1) | ((self.height - 1) << 16), + macro_size: my * uy | ((mx * ux) << 16), + } + } + + pub(crate) fn tiling_registers(&self) -> Option<[(u32, u64); 71]> { + if !self.valid() { + return None; + } + let g = self.geometry(); + let tilemap = self.ta_offset(self.tilemap); + let heapmeta = self.ta_offset(self.heapmeta); + let layermeta = self.ta_offset(self.layer_metadata()); + let target_layers = if self.layers > 1 { + 0xe000 | (self.layers - 1) + } else { + 0x8000 + }; + let deflake_1 = self.ta_offset(self.deflake_1); + let record_index = 0x80001 + self.scene_slot * 4; + Some([ + (0x01748, 1), + (0x10141, 512), + (0x1c039, tilemap), + (0x1c9c8, tilemap), + (0x1c0a1, self.ta_offset(self.tpc)), + (0x1c031, heapmeta | 0x8000000000000000), + (0x1c9c0, heapmeta | 0x8000000000000000), + (0x1c051, 0x3a0012006b0003), + (0x1c061, 1), + (0x10149, self.utile_config), + (0x10139, self.multisample_control), + (0x10111, deflake_1), + (0x1c9b0, deflake_1), + (0x10119, self.ta_offset(self.deflake_2)), + (0x1c9b8, self.ta_offset(self.deflake_2)), + (0x1c958, 1), + (0x1c950, self.ta_offset(self.deflake_3) | 0x4000000000000), + (0x1c930, 0), + (0x1c880, self.ta_offset(self.encoder)), + (0x1c079, layermeta), + (0x1c9d8, layermeta), + (0x10151, 0), + (0x1c199, 0), + (0x1c1a1, 0), + (0x1c1a9, 0), + (0x1c1b1, 0), + (0x1c1b9, 0), + (0x1c8f8, 0x8860), + (0x1c0b1, g.size1), + (0x1c850, g.size1), + (0x10131, 0x88), + (0x10121, self.ppp_control), + (0x10129, g.pixels), + (0x101b9, g.screen), + (0x1c069, g.x_blocks), + (0x1c071, g.y_blocks), + (0x1c081, g.size2), + (0x1c0a9, g.size3), + (0x10171, 256), + (0x10169, target_layers), + (0x0a309, 0), + (0x1c8e0, 0xffffffffffffffff), + (0x1c8e8, 0xffffffffffffffff), + (0x1c898, 0), + (0x101e1, 28), + (0x1c9e8, target_layers & 0x4fff), + (0x1a099, 0), + (0x1a0a1, 0), + (0x1a069, 0), + (0x1a071, 0), + (0x1a0c9, 0), + (0x1a0d1, 0), + (0x101c9, 0), + (0x0d471, 0), + (0x1a0f1, 8), + (0x10799, 0xff0000), + (0x1c830, self.tiling_admission), + (0x1ca30, self.cycle), + (0x16c39, self.cycle), + (0x1c910, record_index), + (0x0a5a1, self.tiling_gate), + (0x0d419, 0x200000001), + (0x1ca10, self.tiling_lifecycle), + (0x014a1, self.tiling_lifecycle), + (0x0a349, self.tiling_lifecycle), + (0x10209, self.tiling_work_stamp), + (0x1c9f0, self.tiling_work_stamp), + (0x14320, self.tiling_work_stamp), + (0x14308, self.completion_control), + (0x14318, self.ta_status | 1), + (0x01740, 1), + ]) + } + + pub(crate) fn fragment_registers(&self) -> Option<[(u32, u64); 87]> { + if !self.valid() { + return None; + } + let g = self.geometry(); + let tile_state = 0x3717f + | ((g.tiles_x - 1) << 44) + | ((g.tiles_y - 1) << 53) + | 0x2000000000 + | if self.layers > 1 { 0x100000000 } else { 0 } + | ((self.utile_config & 0xf000) << 28); + Some([ + (0x01739, 1), + (0x10009, self.utile_config), + (0x15379, self.store_pipeline_bind), + (0x15381, self.store_pipeline), + (0x15369, self.load_pipeline_bind), + (0x15371, self.load_pipeline), + (0x15131, self.merge_upper_x_bits), + (0x15139, self.merge_upper_y_bits), + (0x100a1, 0), + (0x15069, 0), + (0x15071, 0), + (0x16058, 0), + (0x10019, self.multisample_control), + (0x100b1, g.macro_size), + (0x16030, g.macro_size), + (0x100d9, g.screen), + (0x0a301, 0), + (0x10791, 0xff0200), + (0x16098, self.heapmeta), + (0x15109, self.scissor_array), + (0x15101, self.depth_bias_array), + (0x15021, self.aux_fb_flags), + (0x15211, self.height << 32 | self.width), + (0x15049, self.aux_fb_page_count), + (0x10051, self.tib_encoding()), + (0x15321, self.depth_dimensions), + (0x15301, self.depth_clear_value_bits), + (0x15309, self.stencil_clear_value | 0x300), + (0x15311, self.occlusion_query_base), + (0x15319, self.depth_flags), + (0x15349, 0x4040404), + (0x15351, 0), + (0x15329, self.depth_buffer), + (0x15331, self.depth_buffer), + (0x15339, self.stencil_buffer), + (0x15341, self.stencil_buffer), + (0x15231, 0), + (0x15221, 0), + (0x15239, 0), + (0x15229, 0), + (0x15401, self.depth_stride), + (0x15421, self.depth_stride), + (0x15409, self.stencil_stride), + (0x15429, self.stencil_stride), + (0x153c1, self.depth_aux_buffer), + (0x15411, self.depth_aux_stride), + (0x153c9, self.depth_aux_buffer), + (0x15431, self.depth_aux_stride), + (0x153d1, self.stencil_aux_buffer), + (0x15419, self.stencil_aux_stride), + (0x153d9, self.stencil_aux_buffer), + (0x15439, self.stencil_aux_stride), + (0x16429, self.tilemap), + (0x16060, self.layer_metadata()), + (0x16431, 4 * g.size1 << 24), + (0x10039, self.tile_config), + (0x16020, 0), + (0x16451, 0), + (0x15359, 0), + (0x100b8, 0x8860), + (0x16461, self.aux_fb), + (0x16090, self.aux_fb), + (0x101e9, 28), + (0x160a8, 0), + (0x16068, tile_state), + (0x1a0a9, 0), + (0x1a0b1, 0), + (0x1a079, 0), + (0x1a081, 0), + (0x1a0d9, 0), + (0x1a0e1, 0), + (0x101c1, 0), + (0x0d469, 0), + (0x1a0f9, 8), + (0x0a5a9, self.fragment_gate), + (0x0d429, 0x200000001), + (0x160e0, self.fragment_lifecycle), + (0x01499, self.fragment_lifecycle), + (0x0a341, self.fragment_lifecycle), + (0x1c838, self.fragment_admission), + (0x1ca28, self.cycle), + (0x10211, self.fragment_work_stamp), + (0x10420, self.fragment_work_stamp), + (0x14048, self.completion_control), + (0x14080, self.fragment_status | 1), + (0x01731, 1), + (0x01731, 1), + ]) + } +} + +pub(crate) fn pack_registers(records: &[(u32, u64)]) -> Option<[u8; 0x600]> { + if records.len() > 0x600 / 12 { + return None; + } + let mut out = [0; 0x600]; + for (slot, (number, value)) in out.chunks_exact_mut(12).zip(records) { + slot[..4].copy_from_slice(&number.to_le_bytes()); + slot[4..].copy_from_slice(&value.to_le_bytes()); + } + Some(out) +} + +fn u32_at(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn u64_at(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct Addresses { + pub(crate) buffer_manager_block_control: u64, + pub(crate) buffer_manager_block_list: u64, + pub(crate) buffer_manager_counter: u64, + pub(crate) buffer_manager_page_list: u64, + pub(crate) buffer_manager_scene_list: u64, + pub(crate) event_count_array: u64, + pub(crate) fragment_prelude: u64, + pub(crate) pool_slots: u64, + pub(crate) render_shared_state: u64, + pub(crate) tiling_prelude: u64, + + pub(crate) empty_buffer: u64, + pub(crate) event_control: u64, + pub(crate) fragment_counter: u64, + pub(crate) fragment_driver_stamp: u64, + pub(crate) fragment_register_alias: u64, + pub(crate) fragment_shared_tail: u64, + pub(crate) tiling_driver_stamp: u64, + pub(crate) tiling_register_alias: u64, + pub(crate) tiling_shared_tail: u64, + pub(crate) timestamp_end: u64, + pub(crate) timestamp_start: u64, + + pub(crate) buffer_manager: u64, + pub(crate) buffer_manager_slot: u64, + pub(crate) buffer_thing: u64, + pub(crate) context_id: u64, + pub(crate) fragment_event: u64, + pub(crate) fragment_firmware_stamp: u64, + pub(crate) fragment_microsequence: u64, + pub(crate) fragment_queue: u64, + pub(crate) fragment_stamp: u64, + pub(crate) fragment_stats: u64, + pub(crate) fragment_status_page: u64, + pub(crate) fragment_user_timestamp_end: u64, + pub(crate) fragment_user_timestamp_start: u64, + pub(crate) fragment_uuid: u64, + pub(crate) fragment_work: u64, + pub(crate) support: u64, + pub(crate) tiling_counter: u64, + pub(crate) tiling_event: u64, + pub(crate) tiling_firmware_stamp: u64, + pub(crate) tiling_microsequence: u64, + pub(crate) tiling_queue: u64, + pub(crate) tiling_stamp: u64, + pub(crate) tiling_stats: u64, + pub(crate) tiling_status_page: u64, + pub(crate) tiling_user_timestamp_end: u64, + pub(crate) tiling_user_timestamp_start: u64, + pub(crate) tiling_uuid: u64, + pub(crate) tiling_work: u64, +} + +impl Addresses { + pub(crate) fn tiling_microsequence(&self) -> [u8; 0x300] { + let mut out = [0; 0x300]; + let work = self.tiling_work; + let queue = self.tiling_queue; + let user_timestamps = + if self.tiling_user_timestamp_start != 0 || self.tiling_user_timestamp_end != 0 { + work + 0x8d8 + } else { + 0 + }; + u32_at(&mut out, 0x0, 5); + for (offset, value) in [ + (0x14, work + 0x40), + (0x1c, self.buffer_manager), + (0x24, self.buffer_thing), + (0x2c, self.tiling_stats), + (0x34, queue), + (0x3c, work + 0x85c), + ] { + u64_at(&mut out, offset, value); + } + u32_at(&mut out, 0x44, self.context_id as u32); + u32_at(&mut out, 0x48, self.tiling_counter as u32); + u64_at(&mut out, 0x50, self.buffer_manager_slot); + u64_at(&mut out, 0x64, work + 0x784); + u64_at(&mut out, 0x6c, work + 0x8a8); + u32_at(&mut out, 0x7c, self.tiling_uuid as u32); + u64_at(&mut out, 0x190, self.support); + u64_at(&mut out, 0x1a8, self.tiling_status_page); + u64_at(&mut out, 0x1b8, 1); + u32_at(&mut out, 0x1c0, self.tiling_event as u32); + u32_at(&mut out, 0x1cc, 0x80000003); + for (offset, value) in [ + (0x1d0, work + 0x8c0), + (0x1d8, work + 0x8c8), + (0x1e0, work + 0x8c8), + (0x1e8, queue), + (0x1f0, user_timestamps), + (0x1f8, work + 0x928), + (0x200, work + 0x83c), + ] { + u64_at(&mut out, offset, value); + } + u32_at(&mut out, 0x210, self.tiling_uuid as u32); + u32_at(&mut out, 0x218, 1); + u32_at(&mut out, 0x21c, 3); + for (offset, value) in [ + (0x220, work + 0x8c0), + (0x228, work + 0x8c8), + (0x230, work + 0x8d0), + (0x238, queue), + (0x240, user_timestamps), + (0x248, work + 0x928), + (0x250, work + 0x83c), + ] { + u64_at(&mut out, offset, value); + } + u32_at(&mut out, 0x260, self.tiling_uuid as u32); + u32_at(&mut out, 0x268, 6); + for (offset, value) in [ + (0x26c, self.buffer_thing), + (0x274, self.buffer_manager), + (0x27c, self.tiling_stats), + (0x284, queue), + (0x28c, work + 0x85c), + ] { + u64_at(&mut out, offset, value); + } + u32_at(&mut out, 0x294, self.context_id as u32); + u64_at(&mut out, 0x29c, work + 0x784); + u32_at(&mut out, 0x2a8, self.tiling_uuid as u32); + u64_at(&mut out, 0x2b0, self.tiling_firmware_stamp); + u32_at(&mut out, 0x2b8, self.tiling_stamp as u32); + u64_at(&mut out, 0x2e0, self.tiling_microsequence + 0x190); + u32_at(&mut out, 0x2e8, (-0x268i32) as u32); + u64_at(&mut out, 0x2ed, work + 0x918); + u64_at(&mut out, 0x2f5, 0x200000000000001); + u32_at(&mut out, 0x2fc, 0x40000002); + out + } + pub(crate) fn fragment_microsequence(&self) -> [u8; 0x380] { + let mut out = [0; 0x380]; + let work = self.fragment_work; + let queue = self.fragment_queue; + let user_timestamps = + if self.fragment_user_timestamp_start != 0 || self.fragment_user_timestamp_end != 0 { + work + 0xc20 + } else { + 0 + }; + u32_at(&mut out, 0x0, 7); + for (offset, value) in [ + (0x14, work + 0x80), + (0x1c, self.buffer_thing), + (0x24, self.fragment_stats), + (0x2c, work + 0xb70), + (0x34, self.buffer_thing + 0x54), + (0x3c, work + 0xbb0), + (0x44, work + 0xbb4), + (0x4c, queue), + (0x54, work), + ] { + u64_at(&mut out, offset, value); + } + u32_at(&mut out, 0x5c, self.context_id as u32); + u32_at(&mut out, 0x60, self.tiling_counter as u32); + u64_at(&mut out, 0x68, self.buffer_manager_slot); + u64_at(&mut out, 0x7c, work + 0xa58); + u64_at(&mut out, 0x84, work + 0xbf0); + u32_at(&mut out, 0xa0, self.fragment_uuid as u32); + u64_at(&mut out, 0x1b0, self.support); + u64_at(&mut out, 0x1c8, self.fragment_status_page); + u32_at(&mut out, 0x1e0, self.fragment_event as u32); + u32_at(&mut out, 0x1e4, 1); + u32_at(&mut out, 0x1ec, 0x80000003); + for (offset, value) in [ + (0x1f0, work + 0xc08), + (0x1f8, work + 0xc10), + (0x200, work + 0xc10), + (0x208, queue), + (0x210, user_timestamps), + (0x218, work + 0xc70), + (0x220, work + 0xb84), + ] { + u64_at(&mut out, offset, value); + } + u32_at(&mut out, 0x230, self.fragment_uuid as u32); + u32_at(&mut out, 0x238, 1); + u32_at(&mut out, 0x23c, 3); + for (offset, value) in [ + (0x240, work + 0xc08), + (0x248, work + 0xc10), + (0x250, work + 0xc18), + (0x258, queue), + (0x260, user_timestamps), + (0x268, work + 0xc70), + (0x270, work + 0xb84), + ] { + u64_at(&mut out, offset, value); + } + u32_at(&mut out, 0x280, self.fragment_uuid as u32); + u32_at(&mut out, 0x288, 8); + u32_at(&mut out, 0x28c, self.fragment_uuid as u32); + u64_at(&mut out, 0x294, self.fragment_firmware_stamp); + u32_at(&mut out, 0x29c, self.fragment_stamp as u32); + for (offset, value) in [ + (0x2a4, self.buffer_thing), + (0x2ac, self.buffer_manager), + (0x2b4, 1), + (0x2b8, self.fragment_stats), + (0x2c0, work + 0xbb0), + (0x2c8, work + 0xbb4), + (0x2d0, work + 0xb70), + (0x2d8, queue), + (0x2e0, work), + ] { + u64_at(&mut out, offset, value); + } + u32_at(&mut out, 0x2e8, self.context_id as u32); + u64_at(&mut out, 0x2ec, work + 0xa58); + u64_at(&mut out, 0x320, self.fragment_microsequence + 0x1b0); + u32_at(&mut out, 0x328, (-0x288i32) as u32); + u64_at(&mut out, 0x32d, work + 0xc60); + u64_at(&mut out, 0x335, 0x110100000001); + u32_at(&mut out, 0x344, 0x40000002); + out + } +} + +fn tagged_page(address: u64) -> u64 { + (1 << 56) | (address >> 8) +} + +impl Addresses { + fn valid(&self) -> bool { + (self.support | self.tiling_shared_tail | self.fragment_shared_tail) & 0xff == 0 + } +} + +impl Parameters { + pub(crate) fn tiling_work(&self, a: &Addresses) -> Option<[u8; 0x940]> { + if !self.valid() || !a.valid() { + return None; + } + let mut out = [0; 0x940]; + let registers = pack_registers(&self.tiling_registers()?)?; + let g = self.geometry(); + let sampler_array = self.sampler_array; + let sampler_count = self.sampler_count; + let sampler_max = if sampler_count == 0 { + 0 + } else { + sampler_count + 1 + }; + u32_at(&mut out, 0x0, 0x0); + u64_at(&mut out, 0x4, a.tiling_counter); + u32_at(&mut out, 0xc, a.context_id as u32); + u64_at(&mut out, 0x10, a.event_control); + u64_at(&mut out, 0x18, a.buffer_manager_slot); + u64_at(&mut out, 0x20, a.buffer_manager); + u64_at(&mut out, 0x28, a.buffer_thing); + u64_at(&mut out, 0x30, a.empty_buffer); + out[0x40..0x640].copy_from_slice(®isters); + u64_at(&mut out, 0x740, a.tiling_register_alias); + out[0x748..0x74a].copy_from_slice(&(0x41u16).to_le_bytes()); + out[0x74a..0x74c].copy_from_slice(&((0x41 * 0xc) as u16).to_le_bytes()); + u64_at(&mut out, 0x760, self.tpc); + u64_at(&mut out, 0x768, (g.size3 << 0x6) * self.layers); + u64_at(&mut out, 0x770, a.tiling_microsequence); + u32_at(&mut out, 0x778, 0x300); + // Partial rendering finds its matching fragment Work through this + // event/stamp pair, independently of the tiling completion event. + u32_at(&mut out, 0x77c, a.fragment_event as u32); + u32_at(&mut out, 0x780, a.fragment_stamp as u32); + u64_at(&mut out, 0x7a8, self.ta_offset(self.deflake_1)); + out[0x864] = u8::from(self.request_tvb_growth); + u64_at(&mut out, 0x840, self.tiling_lifecycle >> 0x20); + u32_at(&mut out, 0x848, 0xffffffff); + u64_at(&mut out, 0x84c, sampler_array); + u32_at(&mut out, 0x84c + 0x8, sampler_count as u32); + u32_at(&mut out, 0x84c + 0xc, sampler_max as u32); + u64_at(&mut out, 0x878, a.tiling_driver_stamp); + u64_at(&mut out, 0x880, a.tiling_firmware_stamp); + u32_at(&mut out, 0x888, a.tiling_stamp as u32); + u32_at(&mut out, 0x894, 1); // Flush notifier stamps before storage reuse. + u32_at(&mut out, 0x88c, a.tiling_event as u32); + u32_at(&mut out, 0x898, a.tiling_uuid as u32); + u64_at(&mut out, 0x8a0, self.layers - 0x1); + u64_at(&mut out, 0x8c8, a.timestamp_start); + u64_at(&mut out, 0x8d8, a.tiling_user_timestamp_start); + u64_at(&mut out, 0x8e0, a.tiling_user_timestamp_end); + u64_at(&mut out, 0x8ff, tagged_page(a.support)); + u64_at( + &mut out, + 0x908, + 0xdaa0 | if self.layers > 1 { 1 << 62 } else { 0 }, + ); + u64_at(&mut out, 0x910, tagged_page(a.tiling_shared_tail)); + Some(out) + } + pub(crate) fn fragment_work(&self, a: &Addresses) -> Option<[u8; 0xcc0]> { + if !self.valid() || !a.valid() { + return None; + } + let mut out = [0; 0xcc0]; + let registers = pack_registers(&self.fragment_registers()?)?; + let g = self.geometry(); + let sampler_array = self.sampler_array; + let sampler_count = self.sampler_count; + let sampler_max = if sampler_count == 0 { + 0 + } else { + sampler_count + 1 + }; + let tiles_x = g.tiles_x; + let tiles_y = g.tiles_y; + let merge_x = if self.merge_upper_x_bits != 0 { + self.merge_upper_x_bits + } else { + (crate::float::F32::from_bits(0x3fddb3d9) / crate::float::F32::from(self.width as u32)) + .to_bits() as u64 + }; + let merge_y = if self.merge_upper_y_bits != 0 { + self.merge_upper_y_bits + } else { + (crate::float::F32::from_bits(0x3fddb3d9) / crate::float::F32::from(self.height as u32)) + .to_bits() as u64 + }; + let compact_store = (self.store_pipeline & 0xffffffff) << 32; + let compact_partial_store = (self.partial_store_pipeline & 0xffffffff) << 32; + u32_at(&mut out, 0x0, 0x1); + u64_at(&mut out, 0x4, a.fragment_counter); + u32_at(&mut out, 0xc, a.context_id as u32); + u32_at(&mut out, 0x10, 0x0); + u64_at(&mut out, 0x14, a.fragment_microsequence); + u32_at(&mut out, 0x1c, 0x380); + u64_at(&mut out, 0x20, a.event_control); + u64_at(&mut out, 0x28, a.buffer_manager); + u64_at(&mut out, 0x30, a.buffer_thing); + u64_at(&mut out, 0x38, a.empty_buffer); + u64_at(&mut out, 0x40, self.tilemap); + u64_at(&mut out, 0x48, self.multisample_control); + u32_at(&mut out, 0x50, self.samples as u32); + u32_at(&mut out, 0x54, g.macro_size as u32); + u32_at(&mut out, 0x68, merge_x as u32); + u32_at(&mut out, 0x6c, merge_y as u32); + u64_at(&mut out, 0x78, tiles_x * tiles_y); + out[0x80..0x680].copy_from_slice(®isters); + u64_at(&mut out, 0x680, 0x8c020001dddc00); + u64_at(&mut out, 0x688, 0xa0001de1800); + u64_at(&mut out, 0x690, 0x8c020200029800); + u64_at(&mut out, 0x698, 0x220200039520); + u64_at(&mut out, 0x780, a.fragment_register_alias); + out[0x788..0x78a].copy_from_slice(&(0x51u16).to_le_bytes()); + out[0x78a..0x78c].copy_from_slice(&((0x51 * 0xc) as u16).to_le_bytes()); + u64_at(&mut out, 0x7a0, self.depth_bias_array); + u64_at(&mut out, 0x7b0, self.scissor_array); + u64_at(&mut out, 0x7c0, self.occlusion_query_base); + u64_at(&mut out, 0x8f8, self.load_pipeline_bind); + u64_at(&mut out, 0x900, self.load_pipeline); + u64_at(&mut out, 0x928, self.partial_load_pipeline_bind); + u64_at(&mut out, 0x930, self.partial_load_pipeline); + u64_at(&mut out, 0x938, self.depth_flags); + u32_at(&mut out, 0x940, 0x4040404); + u64_at(&mut out, 0x948, self.depth_buffer); + u64_at(&mut out, 0x950, self.depth_stride); + u64_at(&mut out, 0x958, self.depth_aux_stride); + u64_at(&mut out, 0x960, self.depth_buffer); + u64_at(&mut out, 0x968, self.depth_buffer); + u64_at(&mut out, 0x970, self.depth_aux_buffer); + u64_at(&mut out, 0x978, self.stencil_buffer); + u64_at(&mut out, 0x980, self.stencil_stride); + u64_at(&mut out, 0x988, self.stencil_aux_stride); + u64_at(&mut out, 0x990, self.stencil_buffer); + u64_at(&mut out, 0x998, self.stencil_buffer); + u64_at(&mut out, 0x9a0, self.stencil_aux_buffer); + u32_at(&mut out, 0x9b8, self.tib_encoding() as u32); + u64_at(&mut out, 0x9c0, self.aux_fb_flags); + u64_at(&mut out, 0x9c8, (self.height << 0x20) | self.width); + u64_at(&mut out, 0x9d0, self.aux_fb_page_count); + u64_at(&mut out, 0x9d8, self.tile_config); + // The compact EOT records retain the common firmware layout: + // rsrc_spec at +0x0c, USC address at +0x14 (fw/fragment.rs::EotProgram). + u32_at(&mut out, 0x9f4, self.store_pipeline_bind as u32); + u64_at(&mut out, 0x9f8, compact_store); + u64_at(&mut out, 0x9fc, self.store_pipeline); + u32_at(&mut out, 0xa14, self.partial_store_pipeline_bind as u32); + u64_at(&mut out, 0xa18, compact_partial_store); + u64_at(&mut out, 0xa1c, self.partial_store_pipeline); + u32_at(&mut out, 0xa28, self.depth_clear_value_bits as u32); + u32_at( + &mut out, + 0xa2c, + ((self.stencil_clear_value & 0xff) | 0x300) as u32, + ); + u64_at(&mut out, 0xa30, self.sample_size); + u64_at(&mut out, 0xa50, self.depth_dimensions); + u32_at(&mut out, 0xb80, u32::from(self.fragment_sync_grow)); + u64_at(&mut out, 0xb88, self.fragment_lifecycle >> 0x20); + u32_at(&mut out, 0xb90, 0xffffffff); + u64_at(&mut out, 0xb94, sampler_array); + u32_at(&mut out, 0xb94 + 0x8, sampler_count as u32); + u32_at(&mut out, 0xb94 + 0xc, sampler_max as u32); + u32_at(&mut out, 0xba4, u32::from(self.process_empty_tiles)); + u32_at(&mut out, 0xba8, 0x1); + u32_at(&mut out, 0xbac, u32::from(self.samples > 1)); + u64_at(&mut out, 0xbc0, a.fragment_driver_stamp); + u64_at(&mut out, 0xbc8, a.fragment_firmware_stamp); + u32_at(&mut out, 0xbd0, a.fragment_stamp as u32); + u32_at(&mut out, 0xbdc, 1); + u32_at(&mut out, 0xbd4, a.fragment_event as u32); + u32_at(&mut out, 0xbe0, a.fragment_uuid as u32); + u64_at(&mut out, 0xbe8, self.layers - 0x1); + u64_at(&mut out, 0xc10, a.timestamp_start); + u64_at(&mut out, 0xc18, a.timestamp_end); + u64_at(&mut out, 0xc20, a.fragment_user_timestamp_start); + u64_at(&mut out, 0xc28, a.fragment_user_timestamp_end); + u64_at(&mut out, 0xc47, tagged_page(a.support)); + u64_at( + &mut out, + 0xc50, + 0x113a0 | if self.layers > 1 { 1 << 62 } else { 0 }, + ); + u64_at(&mut out, 0xc58, tagged_page(a.fragment_shared_tail)); + u64_at(&mut out, 0xc80, 0x100000000); + Some(out) + } +} + +/// Per-VM firmware operand storage, placed in the caller kernel carveout. +pub(crate) const CONTEXT_BASE: u64 = 0x10_0000_0000; +pub(crate) const PRIVATE_SIZE: usize = 0x35d4000; +pub(crate) const HEAPMETA_OFFSET: u64 = 0x35d0000; +const OPERAND_STRIDE: u64 = 0x208000; + +#[derive(Clone, Copy)] +pub(crate) struct OperandPool { + pub(crate) directory: u64, + pub(crate) table: u64, + buffer: u64, +} +impl OperandPool { + pub(crate) fn new(start: u64, end: u64) -> Option { + let directory = start.checked_add(PRIVATE_SIZE as u64 + 0x4000)?; + let table = directory.checked_add(0x408000)?; + let buffer = table.checked_add(0x18000)?; + let pool = Self { + directory, + table, + buffer, + }; + if start & 0x3fff != 0 || pool.end()? > end { + return None; + } + Some(pool) + } + pub(crate) fn end(&self) -> Option { + self.buffer.checked_add(24 * OPERAND_STRIDE) + } + pub(crate) fn growth_base(&self) -> Option { + Some(self.end()?.checked_add(0x7fff)? & !0x7fff) + } + pub(crate) fn growth_end(&self) -> Option { + self.growth_base()?.checked_add(0x3200000) + } + pub(crate) fn support(&self, id: u64, shared: u64) -> [u8; 0x100] { + let mut out = [0; 0x100]; + u64_at(&mut out, 0, id); + // Render leases use slots 3..59, distinct from opening and compute. + // The native driver assigns +8 before publication; firmware programs + // this slot into the accelerator's private-memory pool selector. + u32_at(&mut out, 8, id as u32); + u64_at(&mut out, 0x14, self.directory); + u32_at(&mut out, 0x1c, 0x400000 / 8); + u32_at(&mut out, 0x24, 24 * 512); + u32_at(&mut out, 0x2c, 24 * 512); + u64_at(&mut out, 0x30, self.table); + u32_at(&mut out, 0x48, 24 * 0x40 / 8); + u64_at(&mut out, 0x4c, shared); + out + } +} + +/// Only accelerator-derived aliases are outside the VM's kernel carveout. +pub(crate) fn fixed_ranges() -> [(u64, usize); 14] { + let mut out = [(0, 0); 14]; + out[..3].copy_from_slice(&[ + (0x1000080000, 0x4000), + (0x1000258000, 0x1c000), + (0x1000240000, 0x4000), + ]); + for i in 0..11 { + out[3 + i] = (0x1000088000 + i as u64 * 0x28000, 0x20000); + } + out +} + +/// (accelerator alias, backing offset in the kernel carveout, extent). +pub(crate) fn private_aliases(pool: &OperandPool) -> [(u64, u64, usize); 40] { + let mut out = [(0, 0, 0); 40]; + out[..5].copy_from_slice(&[ + (0x10_0008_0000, 0x3c000, 0x4000), + (0x10_0025_8000, 0x3450000, 0x1c000), + (0x10_0024_0000, 0x346c000, 0x4000), + (pool.directory, 0x40000, 0x400000), + (pool.table, 0x440000, 0x10000), + ]); + for i in 0..11 { + out[5 + i] = ( + 0x10_0008_8000 + i as u64 * 0x28000, + 0x3470000 + i as u64 * 0x20000, + 0x20000, + ); + } + for i in 0..24 { + out[16 + i] = ( + pool.buffer + i as u64 * OPERAND_STRIDE, + 0x450000 + i as u64 * 0x200000, + 0x200000, + ); + } + out +} + +/// Return one initialized backing page, without allocating a contiguous 54 MiB +/// CPU buffer. All unmapped padding and scratch begin at zero. +pub(crate) fn private_page(pool: &OperandPool, offset: usize, out: &mut [u8]) -> Option<()> { + if offset & 0x3fff != 0 || offset >= PRIVATE_SIZE || out.len() != 0x4000 { + return None; + } + out.fill(0); + if offset == 0x38000 { + u32_at(out, 0x600, 0x60000000); + u32_at(out, 0x604, 0x35b); + } else if (0x40000..0x58000).contains(&offset) { + let first = (offset - 0x40000) / 8; + for i in 0..0x800 { + let page = first + i; + u64_at( + out, + i * 8, + pool.buffer + (page / 512) as u64 * OPERAND_STRIDE + (page % 512) as u64 * 0x1000, + ); + } + } else if offset == 0x440000 { + for i in 0..24 { + u64_at( + out, + i * 0x40, + (pool.buffer + i as u64 * OPERAND_STRIDE) | (1 << 61), + ); + } + } else if offset == 0x3450000 { + for block in 0..11 { + for page in 0..4 { + u32_at( + out, + (block * 4 + page) * 4, + (0x11 + block * 5 + page) as u32, + ); + } + } + } + Some(()) +} + +impl Parameters { + pub(crate) fn set_private(&mut self, start: u64, end: u64) -> Option<()> { + if start & 0x3fff != 0 + || start < CONTEXT_BASE + || start.checked_add(PRIVATE_SIZE as u64)? > end.min(CONTEXT_BASE + (1 << 32)) + { + return None; + } + OperandPool::new(start, end)?; + self.context_base = CONTEXT_BASE; + self.tilemap = start; + self.heapmeta = start + HEAPMETA_OFFSET; + self.tpc = start + 0x28000; + self.deflake_1 = start + 0x2c2a0; + self.deflake_2 = start + 0x2c020; + self.deflake_3 = start + 0x2c000; + self.ta_status = start + 0x30240; + self.fragment_status = start + 0x342c0; + self.aux_fb = start + 0x38000; + Some(()) + } +} + +impl Addresses { + /// Initial device-owned render transport placement. All serialized links + /// are derived from these allocations, independently of client resources. + pub(crate) fn bootstrap() -> Self { + Self { + buffer_manager: 0xfffffc20c0678000, + buffer_manager_block_control: 0xfffffc2000340000, + buffer_manager_block_list: 0xfffffc20c0878000, + buffer_manager_counter: 0xfffffc2000350000, + buffer_manager_page_list: 0xfffffc20c0890000, + buffer_manager_scene_list: 0xfffffc2000348000, + buffer_manager_slot: 0x7, + buffer_thing: 0xfffffc20c0600080, + context_id: 0x9, + empty_buffer: 0xfffffc20c0602800, + event_control: 0xfffffc20c0588100, + event_count_array: 0xfffffc20001b0000, + fragment_counter: 0x0, + fragment_driver_stamp: 0xfffffc2000118044, + fragment_event: 0x11, + fragment_firmware_stamp: 0xfffffc20c0510044, + fragment_microsequence: 0xfffffc20c0469680, + fragment_prelude: 0xfffffc20c029fbc0, + fragment_queue: 0xfffffc20c0824980, + fragment_register_alias: 0x7000141bc0, + fragment_shared_tail: 0xfffffc2000358000, + fragment_stamp: 0x100, + fragment_stats: 0xfffffc20c04db2c8, + fragment_status_page: 0x1000278000, + fragment_user_timestamp_end: 0x0, + fragment_user_timestamp_start: 0x0, + fragment_uuid: 0x900a6ba, + fragment_work: 0xfffffc20c0161b40, + pool_slots: 0x1000080000, + render_shared_state: 0xfffffc20001b8000, + support: 0xfffffc20c05b8000, + tiling_counter: 0x1, + tiling_driver_stamp: 0xfffffc2000118040, + tiling_event: 0x10, + tiling_firmware_stamp: 0xfffffc20c0510040, + tiling_microsequence: 0xfffffc20c03a2340, + tiling_prelude: 0xfffffc20c0350600, + tiling_queue: 0xfffffc20c08224c0, + tiling_register_alias: 0x700004a980, + tiling_shared_tail: 0xfffffc2000300000, + tiling_stamp: 0xc00, + tiling_stats: 0xfffffc20c04da684, + tiling_status_page: 0x1000078000, + tiling_user_timestamp_end: 0x0, + tiling_user_timestamp_start: 0x0, + tiling_uuid: 0x900a6bb, + tiling_work: 0xfffffc20c006a940, + timestamp_end: 0xfffffc2000034fe0, + timestamp_start: 0xfffffc2000034fd8, + } + } +} + +impl Addresses { + pub(crate) fn private_regions(&self) -> [(u64, usize); 9] { + [ + (self.buffer_manager, 0x4000), + (self.buffer_manager_block_control, 0x4000), + (self.buffer_manager_scene_list, 0x4000), + (self.buffer_manager_counter, 0x4000), + (self.buffer_thing & !0x3fff, 0x4000), + (self.buffer_manager_block_list, 0x10000), + (self.buffer_manager_page_list, 0x1c000), + (self.event_control & !0x3fff, 0x4000), + (self.support, 0x4000), + ] + } + + pub(crate) fn private_page(&self, address: u64, out: &mut [u8]) -> Option<()> { + if out.len() != 0x4000 || address & 0x3fff != 0 { + return None; + } + out.fill(0); + if address == self.buffer_manager { + for (off, value) in [ + (0xc, self.buffer_manager_slot as u32), + (0x28, 0x258000), + (0x2c, 0x10), + (0x30, 0x1c000), + (0x34, 0x2c), + (0x38, 0x19b3), + (0x3c, 0xb), + (0x54, 0x2b), + (0x58, 0x20000), + (0x7c, 0x66cc), + (0x80, 0x2244), + (0x84, 0x248000), + ] { + u32_at(out, off, value); + } + for (off, value) in [ + (0x20, self.buffer_manager_page_list), + (0x44, self.buffer_manager_block_list), + (0x4c, self.buffer_manager_block_control), + (0x64, self.buffer_manager_counter), + ] { + u64_at(out, off, value); + } + } else if address == self.buffer_manager_block_control { + u32_at(out, 0, 11); + u32_at(out, 4, 11); + u32_at(out, 0x60, 1); + } else if address == self.buffer_manager_counter { + u32_at(out, 0, 1); + } else if address == self.buffer_manager_block_list { + for i in 0..11 { + u32_at(out, i * 8, (0x11 + i * 5) as u32); + } + } else if address == self.buffer_manager_page_list { + for i in 0..11 { + for j in 0..4 { + u32_at(out, (i * 4 + j) * 4, (0x11 + i * 5 + j) as u32); + } + } + } else if address == self.buffer_thing & !0x3fff { + for i in 0..80 { + for (off, value) in [ + (0, self.pool_slots + i as u64 * 4), + (8, self.buffer_manager_scene_list + i as u64 * 4), + (0x28, 0x240000 + (i as u64 % 36) * 0x30), + (0x40, self.buffer_manager_block_control + 0x40), + (0x48, 0x100000000), + ] { + u64_at(out, i * 0x80 + off, value); + } + } + } else if address == self.event_control & !0x3fff { + for i in 0..36 { + u64_at(out, i * 0x100, self.event_count_array + i as u64 * 4); + } + u32_at(out, (self.event_control as usize & 0x3fff) + 0x10, 0x50); + } + Some(()) + } +} + +/// Reserved support leases; Work storage is allocated per publication. +pub(crate) const SUPPORT_BASE: u64 = 0xffff_fc20_c300_0000; + +impl Addresses { + pub(crate) fn publication(&self, ordinal: u64) -> Self { + let mut a = *self; + a.tiling_counter = a.tiling_counter.wrapping_add(ordinal.wrapping_mul(2)); + a.fragment_counter = a.fragment_counter.wrapping_add(ordinal.wrapping_mul(2)); + a.tiling_stamp = u64::from(crate::g16_stamp::value(ordinal, self.tiling_stamp as u32)); + a.fragment_stamp = u64::from(crate::g16_stamp::value(ordinal, self.fragment_stamp as u32)); + a.buffer_thing = (self.buffer_thing & !0x3fff) + ((ordinal % 80 + 1) % 80) * 0x80; + a + } +} + +impl Addresses { + pub(crate) fn tiling_prelude(&self, blocks: u32) -> [u8; 0x40] { + let mut out = [0; 0x40]; + for (off, value) in [ + (0, 6), + (4, self.context_id as u32), + (8, self.buffer_manager_slot as u32), + (0x10, blocks), + (0x1c, self.tiling_stamp as u32), + ] { + u32_at(&mut out, off, value); + } + u64_at(&mut out, 0x14, self.buffer_manager); + out + } + pub(crate) fn fragment_prelude(&self) -> [u8; 0x40] { + let mut out = [0; 0x40]; + u32_at(&mut out, 0, 4); + u64_at(&mut out, 4, self.tiling_firmware_stamp); + u64_at(&mut out, 0xc, self.tiling_firmware_stamp); + for (off, value) in [ + (0x14, self.tiling_stamp), + (0x20, self.tiling_event), + (0x24, self.fragment_stamp), + (0x28, self.fragment_uuid), + ] { + u32_at(&mut out, off, value as u32); + } + out + } +} + +impl Parameters { + /// Bytes of tilemap and TPC storage required by this pass. The 16 + /// macrotiles each carry their own region and pointer strides. + pub(crate) fn scratch_sizes(&self) -> Option<(usize, usize)> { + if !self.valid() { + return None; + } + let g = self.geometry(); + Some(( + ((g.size1 * 64 * self.layers + 0x3fff) & !0x3fff) as usize, + ((g.size3 * 64 * self.layers + 0x3fff) & !0x3fff) as usize, + )) + } +} diff --git a/drivers/gpu/drm/asahi/g16_runtime.rs b/drivers/gpu/drm/asahi/g16_runtime.rs new file mode 100644 index 0000000000000000000000000000000000000000..93ff250e0097370d7c71fa37b743fa4dd4625826 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_runtime.rs @@ -0,0 +1,1775 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from Niklas Sheth's linux-m4-integration prototype. + +//! G16 ring publication and independent firmware retirement. + +use kernel::io::Io; +use super::Bootstrap; +use crate::{g16_compute as compute, g16_fw, g16_render as render, g16_stamp, g16_vm}; +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::time::{Instant, Monotonic}; +use kernel::{ + c_str, io, + prelude::*, + sync::{Arc, Mutex}, +}; + +/// One userspace queue's three firmware workqueues. Channel rings remain +/// device-wide transports; their messages select these independent queues. +/// Firmware-owned allocations are retained by Bootstrap until device reset. +pub(crate) struct FirmwareQueues { + lanes: Option<[Lane; 3]>, + priority: u32, + events: Option, + pending: usize, + render_count: u64, + compute_count: u64, + pub(crate) last: [Option; 2], +} + +#[derive(Clone, Copy)] +struct Lane { + queue: u64, + pointers: u64, + ring: u64, + head: u32, + new: bool, +} + +impl Lane { + fn firmware_stamp(&self) -> u64 { + self.pointers + 0x1040 + } +} + +/// Reserve three distinct event IDs while this queue has unfinished work. +/// Idle queues release their lease, so creating idle queues does not consume +/// firmware event slots. Slots 0..31 are left to the bootstrap protocol. +struct QueueEvents { + slots: Arc, + index: u32, +} + +impl Drop for QueueEvents { + fn drop(&mut self) { + self.slots.fetch_and(!(1 << self.index), Ordering::AcqRel); + } +} + +impl FirmwareQueues { + pub(crate) fn new(priority: u32) -> Self { + Self { + lanes: None, + priority, + events: None, + pending: 0, + render_count: *crate::module_parameters::fw_counter_start.value(), + compute_count: *crate::module_parameters::fw_counter_start.value(), + last: [None; 2], + } + } + + fn event(&self, stage: u32) -> Result { + Ok(32 + self.events.as_ref().ok_or(EIO)?.index * 3 + stage) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct Stamp { + pub(crate) address: u64, + pub(crate) value: u32, + pub(crate) event: u32, +} +#[derive(Clone, Copy)] +pub(crate) struct Receipt { + pub(crate) id: u64, + pub(crate) stamp: Stamp, +} +struct RenderFlight { + addresses: render::Addresses, + client: Arc>, + root: u64, + events: RenderEvents, + scene: u32, +} +pub(super) struct Flight { + id: u64, + owner: Arc>, + context: u32, + started: Instant, + stamps: [Option; 2], + driver_stamps: [Option; 2], + scratch: g16_vm::Scratch, + client: Arc>, + queues: [(u64, u32); 2], + channels: [(usize, u32); 2], + render: Option, +} + +/// One channel message for all staged commands on this firmware queue. +/// Its epoch remains stable while head advances, so every flight can retain +/// the correct transport-retirement target before the batch is published. +#[derive(Clone, Copy)] +pub(super) struct Publication { + channel: usize, + epoch: u32, + lane: Lane, + event: u32, + sequence: u64, +} + +/// Ring distance is unambiguous because admission keeps less than half a ring +/// outstanding. Equality alone would lose completions when firmware advances +/// past several commands between notifications, including over the wrap boundary. +fn reached(cursor: u32, target: u32, capacity: u32) -> bool { + cursor < capacity && target < capacity && (cursor + capacity - target) % capacity < capacity / 2 +} + +/// Both firmware cursors must leave enough space for the entire publication. +/// The half-ring bound also makes completion comparisons unambiguous. +fn ring_room(tail: u32, read: u32, done: u32, entries: u32, capacity: u32) -> bool { + tail < capacity + && read < capacity + && done < capacity + && entries < capacity / 2 + && [read, done] + .iter() + .all(|&cursor| (tail + capacity - cursor) % capacity + entries < capacity / 2) +} + +impl Bootstrap { + /// Report each exhausted resource once, without logging every retry or + /// enabling firmware KTrace (which changes command execution timing). + fn report_pressure(&self, bit: u64, resource: &str) { + if *crate::module_parameters::fw_trace.value() & 2 != 0 + && self.pressure_reported.fetch_or(bit, Ordering::Relaxed) & bit == 0 + { + dev_info!( + self.dev.as_ref(), + "G16: backpressure {} outstanding={}\n", + resource, + self.flights.len() + ); + } + } + + /// Lease Work storage until notifier/driver stamps and transport retire. + /// Backing stays mapped. Queue-owned wrapping stamps outlive reused Work, + /// so a pending dependency never follows a recycled completion address. + fn allocate_work(&mut self, render: bool) -> Result { + use crate::pgtable::prot::{PROT_FW_PRIV_RW, PROT_FW_SHARED_RW}; + let index = usize::from(render); + if let Some(va) = self.free_work[index].pop() { + return Ok(va); + } + // Retirement must not allocate. Space for every outstanding lease + // also covers all of them completing before another acquisition. + self.free_work[index].reserve(self.flights.len() + 1, GFP_KERNEL)?; + let stride = if render { 0x1c000 } else { 0x8000 }; + if self.work_arenas[index].is_empty() { + let base = self.next_work_va; + let end = base.checked_add(16 * stride).ok_or(ENOSPC)?; + // Failed refills retain their partial allocations, but no part of + // that range becomes available for a later Work or another arena. + self.next_work_va = end; + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + for va in (base..end).step_by(stride as usize) { + fw.alloc(va, if render { 0x10000 } else { 0x4000 }, PROT_FW_PRIV_RW)?; + if render { + fw.alloc(va + 0x10000, 0xc000, PROT_FW_SHARED_RW)?; + } else { + fw.alloc(va + 0x4000, 0x4000, PROT_FW_SHARED_RW)?; + } + } + fw.sync(); + self.work_arenas[index] = base..end; + } + let va = self.work_arenas[index].start; + self.work_arenas[index].start += stride; + Ok(va) + } + + fn prepare_queues(&mut self, owner: &mut FirmwareQueues) -> Result { + use crate::pgtable::prot::{PROT_FW_PRIV_RW, PROT_FW_SHARED_RW}; + use g16_fw::queue as q; + + if owner.lanes.is_none() { + let base = self.next_work_va; + self.next_work_va = base.checked_add(0x24000).ok_or(ENOSPC)?; + let lanes = core::array::from_fn(|stage| { + let queue = base + stage as u64 * 0xc000; + Lane { + queue, + pointers: queue + 0x4000, + ring: queue + 0x8000, + head: 0, + new: true, + } + }); + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + // A notifier list and GPU context belong to the public queue, + // shared only by its own vertex, fragment and compute lanes. + let jobs = base + 0x4100; + let context = base + 0x4140; + for (stage, lane) in lanes.iter().enumerate() { + fw.alloc(lane.queue, 0x4000, PROT_FW_PRIV_RW)?; + fw.alloc(lane.pointers, 0x8000, PROT_FW_SHARED_RW)?; + let mut descriptor = q::Queue { + pointers: lane.pointers, + ring: lane.ring, + jobs, + private: lane.queue + 0xb0, + context, + uuid: ((lane.queue - 0xffff_fc22_0000_0000) >> 14) as u32 + 2, + } + .encode_priority(owner.priority) + .ok_or(EINVAL)?; + // Native TA/3D profiles allow firmware to suspend and restart + // renders. Their Work/scratch leases survive until the final + // notifier/driver stamp, including intervening partial runs. + // Compute remains non-preemptible, matching the M1/M2 driver. + if stage == 2 { + descriptor[0x30..0x34].copy_from_slice(&0u32.to_le_bytes()); + descriptor[0x48..0x4c].copy_from_slice(&1u32.to_le_bytes()); + } + fw.write(lane.queue, &descriptor)?; + fw.write(lane.pointers, &q::pointers(0x500, 0))?; + // A newly created timeline starts immediately before its first + // target. This also permits qualification near stamp rollover. + let (ordinal, first) = match stage { + 0 => (owner.render_count, 0xc00), + 1 => (owner.render_count, 0x100), + _ => (owner.compute_count, 0x100), + }; + let current = g16_stamp::previous(g16_stamp::value(ordinal, first)); + fw.write_live(lane.firmware_stamp(), ¤t.to_le_bytes())?; + } + fw.write(jobs, &q::jobs(jobs))?; + fw.write(context, &q::context())?; + fw.sync(); + owner.lanes = Some(lanes); + if *crate::module_parameters::fw_trace.value() & 1 != 0 { + dev_info!( + self.dev.as_ref(), + "G16: queues vertex={:#x} fragment={:#x} compute={:#x} priority={}\n", + lanes[0].queue, + lanes[1].queue, + lanes[2].queue, + owner.priority + ); + } + } + if owner.events.is_none() { + let previous = self + .queue_events + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |mask| { + let free = !mask & 0xffff_ffff; + (free != 0).then(|| mask | (1 << free.trailing_zeros())) + }) + .map_err(|_| { + self.report_pressure(8, "events"); + EBUSY + })?; + owner.events = Some(QueueEvents { + slots: self.queue_events.clone(), + index: (!previous & 0xffff_ffff).trailing_zeros(), + }); + } + Ok(()) + } + + fn reserve_flight( + &mut self, + queue: &FirmwareQueues, + render: bool, + root: u64, + entries: [u32; 3], + ) -> Result { + if self.render_failed { + return Err(EIO); + } + // Check scarce resources before allocating any per-Work storage. + // Commands in either persistent VM root can share its live ASID. + if !(1..64).any(|i| { + self.context_users[i] == 0 || self.context_roots[i].is_some_and(|s| s.low == root) + }) { + self.report_pressure(1, "ASIDs"); + return Err(EBUSY); + } + let stages: &[usize] = if render { &[0, 1] } else { &[2] }; + let config = g16_fw::MainConfig::bootstrap(g16_fw::BUNDLE_ADDRESS); + let guard = self._sgx.try_access().ok_or(ENODEV)?; + for &stage in stages { + let channel = queue.priority as usize * 3 + stage; + let route = &config.channels[channel]; + let read = + guard.try_read32(0xd64000 + (route.state[0] - 0xffff_fc20_0002_8000) as usize)?; + let done = + guard.try_read32(0xd64000 + (route.state[1] - 0xffff_fc20_0002_8000) as usize)?; + let tail = + guard.try_read32(0xd60000 + (route.state[2] - 0xffff_fc20_0002_0000) as usize)?; + if read >= 256 || done >= 256 || tail >= 256 { + return Err(EIO); + } + let staged = self + .publications + .iter() + .filter(|p| p.channel == channel) + .count() as u32; + let coalesced = queue.lanes.as_ref().is_some_and(|lanes| { + self.publications + .iter() + .any(|p| p.lane.queue == lanes[stage].queue) + }); + if !ring_room( + (tail + staged) & 255, + read, + done, + u32::from(!coalesced), + 256, + ) { + self.report_pressure(2, "channel"); + return Err(EBUSY); + } + // Bound transport epochs, not the number of Works sharing each + // epoch. A coalesced batch consumes only one channel-ring slot. + let next = (tail + staged + u32::from(!coalesced)) & 255; + if self.flights.iter().any(|f| { + f.channels + .iter() + .any(|&(c, target)| c == channel && !reached(next, target, 256)) + }) { + self.report_pressure(2, "channel"); + return Err(EBUSY); + } + if let Some(lanes) = &queue.lanes { + let lane = lanes[stage]; + let fw = self._firmware_space.as_ref().ok_or(EIO)?; + let read = fw.read_u32(lane.pointers)?; + let done = fw.read_u32(lane.pointers + 0x30)?; + if read >= 0x500 || done >= 0x500 || lane.head >= 0x500 { + return Err(EIO); + } + if !ring_room(lane.head, read, done, entries[stage], 0x500) { + self.report_pressure(16, "workqueue"); + return Err(EBUSY); + } + } + } + drop(guard); + self.flights.reserve(1, GFP_KERNEL)?; + self.publications.reserve(stages.len(), GFP_KERNEL)?; + Ok(()) + } + + /// Stage a queue head; multiple Works share one transport message. + fn stage_queue( + &mut self, + channel: usize, + lane: Lane, + event: u32, + sequence: u64, + ) -> Result { + if let Some(p) = self + .publications + .iter_mut() + .find(|p| p.lane.queue == lane.queue) + { + p.lane.head = lane.head; + p.sequence = sequence; + return Ok(p.epoch); + } + let config = g16_fw::MainConfig::bootstrap(g16_fw::BUNDLE_ADDRESS); + let route = &config.channels[channel]; + let guard = self._sgx.try_access().ok_or(ENODEV)?; + let tail = + guard.try_read32(0xd60000 + (route.state[2] - 0xffff_fc20_0002_0000) as usize)?; + if tail >= 256 { + return Err(EIO); + } + drop(guard); + let reserved = self + .publications + .iter() + .filter(|p| p.channel == channel) + .count() as u32; + let epoch = (tail + reserved) & 255; + self.publications.push( + Publication { + channel, + epoch, + lane, + event, + sequence, + }, + GFP_KERNEL, + )?; + Ok(epoch) + } + + /// Publish fully prepared rings together, then kick each used engine once + /// per priority. No staged Work may remain when the worker sleeps. + pub(crate) fn flush(&mut self) -> Result { + if self.publications.is_empty() { + return Ok(()); + } + self.render_failed = true; + let config = g16_fw::MainConfig::bootstrap(g16_fw::BUNDLE_ADDRESS); + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + let mut tails = [None; 12]; + let mut kicks = 0u32; + // Fragment transports must precede tiling, matching normal publication. + for stage in [1usize, 0, 2] { + for p in self.publications.iter().filter(|p| p.channel % 3 == stage) { + let route = &config.channels[p.channel]; + fw.write_live(p.lane.pointers + 0x40, &p.lane.head.to_le_bytes())?; + fw.write_live( + route.ring + u64::from(p.epoch) * 0x18, + &g16_fw::queue::channel( + p.lane.queue, + p.lane.head as u16, + p.event as u8, + stage as u32, + p.lane.new, + p.sequence, + ), + )?; + tails[p.channel] = Some((p.epoch + 1) & 255); + kicks |= 1 << ((p.channel / 3) * 2 + usize::from(stage == 2)); + } + } + crate::mem::sync(); + let guard = self._sgx.try_access().ok_or(ENODEV)?; + for stage in [1usize, 0, 2] { + for channel in (stage..12).step_by(3) { + if let Some(tail) = tails[channel] { + let route = &config.channels[channel]; + guard.try_write32( + tail, + 0xd60000 + (route.state[2] - 0xffff_fc20_0002_0000) as usize, + )?; + } + } + } + crate::mem::sync(); + drop(guard); + for bit in 0..8 { + if kicks & (1 << bit) != 0 { + let priority = bit / 2; + let engine = (bit % 2) * 2; + self._rtkit.as_mut().send_message( + 0x21, + 0x0083_0000_0000_0000 | ((priority as u64) << 2) | engine as u64, + )?; + } + } + if *crate::module_parameters::fw_trace.value() & 4 != 0 { + dev_info!( + self.dev.as_ref(), + "G16: batch works={} messages={} kicks={}\n", + self.batch_works, + self.publications.len(), + kicks.count_ones() + ); + } + self.publications.clear(); + self.batch_works = 0; + self.render_failed = false; + Ok(()) + } + + pub(crate) fn poison(&mut self) { + self.render_failed = true; + } + + fn stamps_done(&self, flight: &Flight) -> Result { + let fw = self._firmware_space.as_ref().ok_or(EIO)?; + for stamp in flight.stamps.iter().flatten() { + if !g16_stamp::reached(fw.read_u32(stamp.address)?, stamp.value) { + return Ok(false); + } + } + Ok(true) + } + + fn trace_publication(&self, kind: &str, id: u64) -> Result { + if *crate::module_parameters::fw_trace.value() & 1 != 0 { + let mut unretired = 0; + for flight in &self.flights { + if !self.stamps_done(flight).unwrap_or(false) { + unretired += 1; + } + } + let flight = self.flights.iter().find(|f| f.id == id).ok_or(EIO)?; + dev_info!( + self.dev.as_ref(), + "G16: publish {} {:#x} outstanding={} unfinished_stamps={} queues={:#x}/{:#x} events={}/{}\n", + kind, + id, + self.flights.len(), + unretired, + flight.queues[0].0.saturating_sub(0x4000), + flight.queues[1].0.saturating_sub(0x4000), + flight.stamps[0].map_or(u32::MAX, |s| s.event), + flight.stamps[1].map_or(u32::MAX, |s| s.event) + ); + } + Ok(()) + } + + /// Optional qualification trace. Readiness is observed only for diagnostics; + /// it does not decide whether a dependent Work can enter the firmware ring. + fn trace_dependencies( + &self, + kind: &str, + id: u64, + context: u32, + dependencies: &[Option; 2], + ) -> Result { + if *crate::module_parameters::fw_trace.value() & 8 != 0 { + let fw = self._firmware_space.as_ref().ok_or(EIO)?; + let mut unfinished = 0; + for dep in dependencies.iter().flatten() { + unfinished += usize::from(!g16_stamp::reached(fw.read_u32(dep.address)?, dep.value)); + } + let flight = self.flights.iter().find(|flight| flight.id == id).ok_or(EIO)?; + let mut driver_values = [0; 2]; + for (index, stamp) in flight.driver_stamps.iter().enumerate() { + if let Some(stamp) = stamp { + driver_values[index] = fw.read_u32(stamp.address)?; + } + } + dev_info!( + self.dev.as_ref(), + "G16: staged {} {:#x} context={} dependencies={} unfinished={} stamps={:#x}/{:#x} driver_stamps={:#x}/{:#x}\n", + kind, + id, + context, + dependencies.iter().flatten().count(), + unfinished, + flight.stamps[0].map_or(0, |stamp| stamp.value), + flight.stamps[1].map_or(0, |stamp| stamp.value), + driver_values[0], + driver_values[1] + ); + } + Ok(()) + } + + /// Preserve the event and unfinished queue identities before poisoning the + /// device. Firmware timeout events name a stamp slot, which can be shared + /// by several in-flight Works from the same userspace queue. + fn report_failed_event(&self, base: u64, kind: u32, cursor: u32) -> Result { + let fw = self._firmware_space.as_ref().ok_or(EIO)?; + dev_err!(self.dev.as_ref(), "G16: failed Event kind={} cursor={} flights={}\n", + kind, cursor, self.flights.len()); + for offset in (0..0x48).step_by(8) { + dev_err!(self.dev.as_ref(), "G16: Event +{:#x}={:#018x}\n", + offset, fw.read_u64(base + offset)?); + } + for flight in self.flights.iter().take(16) { + dev_err!(self.dev.as_ref(), "G16: pending Work={:#x} context={} age_ms={} render={}\n", + flight.id, flight.context, flight.started.elapsed().as_millis(), + flight.render.is_some()); + for (stage, stamp) in flight.stamps.iter().enumerate() { + if let Some(stamp) = stamp { + let (pointers, head) = flight.queues[stage]; + dev_err!(self.dev.as_ref(), + "G16: stage={} event={} stamp={:#x} actual={:#x} expected={:#x} pointers={:#x} read={} done={} target={}\n", + stage, stamp.event, stamp.address, fw.read_u32(stamp.address)?, + stamp.value, pointers, fw.read_u32(pointers)?, + fw.read_u32(pointers + 0x30)?, head); + } + } + } + Ok(()) + } + + /// Dispatch events by the outstanding Work identity, never by the most + /// recently submitted job. Completion notifications are hints; stamps and + /// ring retirement remain authoritative even if event bits coalesce. + fn service_flights(&mut self) -> Result { + const STATE: u64 = 0xffff_fc20_0003_f1c0; + const RING: u64 = 0xffff_fc20_0003_f440; + for _ in 0..256 { + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + let cursor = fw.read_u32(STATE)?; + let end = fw.read_u32(STATE + 0x20)?; + if cursor >= 256 || end >= 256 { + return Err(EIO); + } + if cursor == end { + break; + } + let base = RING + u64::from(cursor) * 0x48; + let kind = fw.read_u32(base)?; + if kind == 1 + || (kind == 13 + && self.opening_notification_pending + && fw.read_u32(base + 4)? == 1 + && fw.read_u32(base + 8)? == 0 + && fw.read_u64(base + 12)? == g16_fw::CONTROL_DATA + && fw.read_u64(base + 20)? == g16_fw::OPERAND_TABLE + && fw.read_u64(base + 28)? == g16_fw::OPERAND_TABLE + && fw.read_u32(base + 36)? == 20 * 8) + { + if kind == 13 { + self.opening_notification_pending = false; + } + fw.write_live(STATE, &((cursor + 1) & 255).to_le_bytes())?; + continue; + } + let mut selected = None; + for (index, flight) in self.flights.iter().enumerate() { + if let Some(render) = &flight.render { + let a = &render.addresses; + let matches = match kind { + 6 => { + fw.read_u32(base + 4)? == a.context_id as u32 + && fw.read_u32(base + 8)? == a.buffer_manager_slot as u32 + && fw.read_u32(a.fragment_firmware_stamp)? + != a.fragment_stamp as u32 + } + 7 => { + fw.read_u32(base + 12)? == a.fragment_event as u32 + && fw.read_u32(base + 16)? == a.fragment_stamp as u32 + } + _ => false, + }; + if matches { + selected = Some(index); + break; + } + } + } + let Some(index) = selected else { + self.report_failed_event(base, kind, cursor)?; + return Err(EIO); + }; + let mut render = self.flights[index].render.take().ok_or(EIO)?; + let pool_index = self + .tvb_pools + .iter() + .position(|p| p.root == render.root) + .ok_or(EIO)?; + let mut pool = self.tvb_pools.swap_remove(pool_index); + let result = self.service_render_events( + &mut render.client.lock(), + &mut pool, + &render.addresses, + &mut render.events, + ); + self.tvb_pools.push(pool, GFP_KERNEL)?; + self.flights[index].render = Some(render); + result?; + if self._firmware_space.as_ref().ok_or(EIO)?.read_u32(STATE)? == cursor { + break; + } + } + self.drain_trace_rx() + } + + pub(crate) fn retire(&mut self) -> Result> { + if self.render_failed { + return Err(EIO); + } + self.service_flights().inspect_err(|error| { + dev_err!( + self.dev.as_ref(), + "G16: firmware event service failed: {:?}\n", + error + ); + })?; + let config = g16_fw::MainConfig::bootstrap(g16_fw::BUNDLE_ADDRESS); + for index in 0..self.flights.len() { + let flight = &self.flights[index]; + if !self.stamps_done(flight)? { + continue; + } + let fw = self._firmware_space.as_ref().ok_or(EIO)?; + let mut done = true; + // The notifier pass drops embedded JobMeta references and cleans + // their cache lines before publishing these later driver stamps. + for stamp in flight.driver_stamps.iter().flatten() { + done &= g16_stamp::reached(fw.read_u32(stamp.address)?, stamp.value); + } + // A reusable Work has its own notifier and exact submitted count. + // Removal must be visible before that storage can change queues. + let notifier = flight.id + + if flight.render.is_some() { + 0x200 + } else { + 0x1400 + }; + done &= fw.read_u32(notifier + 0x94)? == 0; + for &(pointers, head) in &flight.queues { + if pointers != 0 { + done &= reached(fw.read_u32(pointers)?, head, 0x500) + && reached(fw.read_u32(pointers + 0x30)?, head, 0x500); + } + } + let guard = self._sgx.try_access().ok_or(ENODEV)?; + for &(channel, target) in &flight.channels { + if channel == usize::MAX { + continue; + } + for address in &config.channels[channel].state[..2] { + done &= reached( + guard.try_read32(0xd64000 + (*address - 0xffff_fc20_0002_8000) as usize)?, + target, + 256, + ); + } + } + // Growth replies can refer to a Work after its main stage stamp. + if flight.render.is_some() { + done &= guard.try_read32(0xd640c0)? == guard.try_read32(0xd60060)? + && guard.try_read32(0xd640c8)? == guard.try_read32(0xd60060)?; + } + drop(guard); + if !done { + continue; + } + let limited = flight + .render + .as_ref() + .is_some_and(|r| r.events.memory_limit); + let id = flight.id; + let render_work = flight.render.is_some(); + if let Some(render) = &flight.render { + let pool = self + .tvb_pools + .iter_mut() + .find(|p| p.root == render.root) + .ok_or(EIO)?; + pool.release_scene(render.scene); + } + let mut owner = flight.owner.lock(); + owner.pending -= 1; + if owner.pending == 0 { + owner.events = None; + } + drop(owner); + self.context_users[flight.context as usize] -= 1; + let flight = self.flights.remove(index).map_err(|_| EIO)?; + flight.client.lock().release_scratch(flight.scratch)?; + self.free_work[usize::from(render_work)].push(id, GFP_KERNEL)?; + // Queued commands can legitimately wait longer than one Work's + // timeout. Watch for a lack of retirement progress, rather than + // charging every command for all its predecessors' execution. + let now = Instant::now(); + for pending in &mut self.flights { + pending.started = now; + } + return Ok(Some((id, if limited { Err(ENOMEM) } else { Ok(()) }))); + } + if let Some(flight) = self + .flights + .iter() + .find(|f| f.started.elapsed().as_millis() >= 10000) + { + dev_err!( + self.dev.as_ref(), + "G16: async Work {:#x} context {} timeout without retirement progress\n", + flight.id, + flight.context + ); + self.render_failed = true; + return Err(ETIMEDOUT); + } + Ok(None) + } + + /// Notifications do not extend the watchdog. Only retirement progress + /// resets it, so trace traffic cannot keep a stuck command alive. + pub(crate) fn watchdog_remaining_ms(&self) -> u32 { + self.flights + .iter() + .map(|flight| (10000 - flight.started.elapsed().as_millis()).clamp(1, 10000) as u32) + .min() + .unwrap_or(10000) + } +} + +/// A per-VM firmware support object retained by every queued execution until +/// its GPU accesses finish, including after public VM/queue handle destruction. +pub(crate) struct Support { + slots: Arc, + index: u32, + growth_base: u64, + submissions: AtomicU64, +} +impl Support { + pub(crate) fn address(&self) -> u64 { + render::SUPPORT_BASE + u64::from(self.index) * 0x4000 + } +} +impl Drop for Support { + fn drop(&mut self) { + self.slots.fetch_and(!(1 << self.index), Ordering::AcqRel); + } +} + +impl Bootstrap { + pub(crate) fn reserve_timestamp(&mut self, size: u64) -> Result { + if size == 0 || size & 0x3fff != 0 { + return Err(EINVAL); + } + // Match the shim's firmware-shared special-object aperture. + let mut base = g16_fw::TIMESTAMP_BASE; + for range in &self.timestamp_ranges { + if base.checked_add(size).ok_or(ENOSPC)? <= range.start { + break; + } + base = base.max(range.end); + } + let end = base.checked_add(size).ok_or(ENOSPC)?; + if end > g16_fw::TIMESTAMP_BASE + g16_fw::TIMESTAMP_SIZE { + return Err(ENOSPC); + } + self.timestamp_ranges.push(base..end, GFP_KERNEL)?; + self.timestamp_ranges.sort_unstable_by_key(|r| r.start); + Ok(base) + } + + pub(crate) fn map_timestamp(&mut self, address: u64, physical: u64, size: usize) -> Result { + self._firmware_space.as_mut().ok_or(EIO)?.map_external( + address, + physical, + size, + crate::pgtable::prot::PROT_FW_SHARED_RW, + )?; + self._firmware_space.as_ref().ok_or(EIO)?.sync(); + Ok(()) + } + pub(crate) fn release_timestamp(&mut self, range: core::ops::Range) -> Result { + self._firmware_space + .as_mut() + .ok_or(EIO)? + .unmap_external(range.clone())?; + self.timestamp_ranges.retain(|r| *r != range); + Ok(()) + } + + pub(crate) fn render_failed(&self) -> bool { + self.render_failed + } + + pub(crate) fn acquire_render_support(&mut self, pool: &render::OperandPool) -> Result { + if self.render_failed { + return Err(EIO); + } + let mut used = self.render_support.load(Ordering::Acquire); + let index = loop { + let index = (!used).trailing_zeros(); + if index >= 57 { + return Err(ENOSPC); + } + match self.render_support.compare_exchange_weak( + used, + used | (1 << index), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break index, + Err(value) => used = value, + } + }; + let lease = Support { + slots: self.render_support.clone(), + index, + growth_base: pool.growth_base().ok_or(EINVAL)?, + submissions: AtomicU64::new( + crate::module_parameters::fw_counter_start.value().wrapping_mul(2), + ), + }; + let mut a = render::Addresses::bootstrap(); + a.support = lease.address(); + a.render_shared_state = lease.address() + 0x1000; + let mut page = KVec::new(); + page.extend_with(0x4000, 0u8, GFP_KERNEL)?; + page[..0x100].copy_from_slice(&pool.support(3 + u64::from(index), a.render_shared_state)); + self._firmware_space + .as_mut() + .ok_or(EIO)? + .write_live(a.support, &page)?; + Ok(lease) + } + + fn set_context(&mut self, index: usize, roots: Option) -> Result { + let res = self + .dev + .as_ref() + .of_node() + .ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("ttbs"))?; + // SAFETY: The reserved table has 64 two-root slots. A slot is changed + // only with no outstanding Work using it, under the runtime mutex. + let table = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + unsafe { + let ptr = table.ptr().add(index * 16); + ptr.cast::() + .write_volatile(ptr.cast::().read_volatile() & !1); + ptr.add(8) + .cast::() + .write_volatile(ptr.add(8).cast::().read_volatile() & !1); + core::arch::asm!("dc cvac, {addr}", "dsb oshst", addr=in(reg) ptr); + crate::mem::tlbi_asid(index as u8); + core::arch::asm!("dsb osh", "isb"); + if let Some(roots) = roots { + ptr.cast::() + .write_volatile(roots.low | ((index as u64) << 48) | 1); + ptr.add(8) + .cast::() + .write_volatile(roots.high | ((index as u64) << 48) | 1); + core::arch::asm!("dc cvac, {addr}", addr=in(reg) ptr); + } + } + crate::mem::sync(); + self.context_roots[index] = roots; + Ok(()) + } + + /// Invalidate only slots actually bound to this root. New roots need no + /// shootdown: set_context invalidates the ASID before installing them. + pub(crate) fn invalidate_client(&self, client: &g16_vm::AddressSpace) { + let root = client.roots().low; + if self + ._address_space + .as_ref() + .is_some_and(|s| s.roots().low == root) + { + client.low.invalidate(Some(0)); + client.high.invalidate(Some(0)); + } + for (index, roots) in self.context_roots.iter().enumerate().skip(1) { + if roots.is_some_and(|r| r.low == root) { + client.low.invalidate(Some(index as u8)); + client.high.invalidate(Some(index as u8)); + } + } + crate::mem::sync(); + client.low.clear_invalidations(); + client.high.clear_invalidations(); + } + + fn acquire_context(&mut self, roots: g16_vm::Roots, preferred: usize) -> Result { + let existing = (1..64).find(|&i| self.context_roots[i].is_some_and(|r| r.low == roots.low)); + let slot = existing + .or_else(|| { + if self.context_users[preferred] == 0 { + Some(preferred) + } else { + (1..64).find(|&i| self.context_users[i] == 0) + } + }) + .ok_or(EBUSY)?; + if existing.is_none() { + self.set_context(slot, Some(roots))?; + } + self.context_users[slot] += 1; + Ok(slot as u32) + } + + pub(crate) fn release_render_vm(&mut self, space: g16_vm::Roots) -> Result { + if self.render_failed { + return Err(EIO); + } + for index in 1..64 { + if self.context_roots[index].is_some_and(|r| r.low == space.low) { + if self.context_users[index] != 0 { + return Err(EBUSY); + } + self.set_context(index, None)?; + } + } + if let Some(index) = self.tvb_pools.iter().position(|p| p.root == space.low) { + let a = self.tvb_pools[index].addresses; + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + fw.write_live(0xfffffc2000128000 + a.buffer_manager_slot * 16, &[0; 16])?; + fw.release_region(a.buffer_manager)?; + self.tvb_pools.swap_remove(index); + } + Ok(()) + } + + fn drain_trace_rx(&mut self) -> Result { + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + for (state, capacity) in [ + (0xffff_fc20_0003_f3c0, 512), + (0xffff_fc20_0003_f400, 256), + (0xffff_fc20_0003_f200, 256), + (0xffff_fc20_0003_f230, 256), + (0xffff_fc20_0003_f260, 256), + (0xffff_fc20_0003_f290, 256), + (0xffff_fc20_0003_f2c0, 256), + (0xffff_fc20_0003_f2f0, 256), + ] { + let read = fw.read_u32(state)?; + let write = fw.read_u32(state + 0x20)?; + if read >= capacity || write >= capacity { + dev_err!( + self.dev.as_ref(), + "G16: invalid RX cursors at {:#x}: {}/{} capacity {}\n", + state, + read, + write, + capacity + ); + return Err(EIO); + } + if read != write { + if state == 0xffff_fc20_0003_f3c0 { + let mut cursor = read; + while cursor != write { + let base = 0xffff_fc20_000e_5c40 + u64::from(cursor) * 0x48; + if fw.read_u32(base)? == 5 + && fw.read_u32(base + 0x2c)? & 0xffff == 4 + && fw.read_u64(base + 0x30)? == 0 + && fw.read_u64(base + 0x14)? != 0 + { + pr_info!( + "G16: firmware Start3D partial render queue {:#x}\n", + fw.read_u64(base + 0x24)? + ); + } + cursor = (cursor + 1) % capacity; + } + } + fw.write_live(state, &write.to_le_bytes())?; + } + } + Ok(()) + } + + /// The caller retains all client mappings throughout this call. On failure + /// after publication it must quarantine those mappings until reset. + pub(crate) fn submit_render( + &mut self, + queues: Arc>, + owner: Arc>, + params: &render::Parameters, + support: &Support, + timestamps: [u64; 4], + dependencies: &[Option; 2], + ) -> Result { + let mut queue = queues.lock(); + let root = owner.lock().roots().low; + let initbm = !self + .tvb_pools + .iter() + .any(|p| p.root == root && p.initialized); + self.reserve_flight( + &queue, + true, + root, + [ + 1 + u32::from(initbm) + dependencies.iter().flatten().count() as u32, + 2, + 0, + ], + )?; + self.prepare_queues(&mut queue)?; + let result = self.submit_render_queue( + &mut queue, + queues.clone(), + owner, + params, + support, + timestamps, + dependencies, + ); + if queue.pending == 0 { + queue.events = None; + } + result + } + + fn submit_render_queue( + &mut self, + queue: &mut FirmwareQueues, + queues: Arc>, + owner: Arc>, + params: &render::Parameters, + support: &Support, + timestamps: [u64; 4], + dependencies: &[Option; 2], + ) -> Result { + let mut guard = owner.lock(); + let client = &mut *guard; + let root = client.roots().low; + let index = match self.tvb_pools.iter().position(|p| p.root == root) { + Some(index) => index, + None => { + self.tvb_pools.reserve(1, GFP_KERNEL)?; + let base = self.next_work_va; + self.next_work_va = base.checked_add(0x44000).ok_or(ENOSPC)?; + let pool = crate::g16_tvb::Tvb::new( + self._firmware_space.as_mut().ok_or(EIO)?, + client, + base, + 7 + u64::from(support.index), + support.growth_base, + )?; + self.tvb_pools.push(pool, GFP_KERNEL)?; + self.tvb_pools.len() - 1 + } + }; + let scene = self.tvb_pools[index].reserve_scene().inspect_err(|_| { + self.report_pressure(4, "scenes"); + })?; + let mut pool = self.tvb_pools.swap_remove(index); + let result = self.submit_render_pool( + client, + params, + support, + queue, + queues, + &mut pool, + owner.clone(), + scene, + timestamps, + dependencies, + ); + if result.is_err() && !self.render_failed { + pool.release_scene(scene); + } + self.tvb_pools.push(pool, GFP_KERNEL)?; + result + } + + fn submit_render_pool( + &mut self, + client: &mut g16_vm::AddressSpace, + params: &render::Parameters, + support: &Support, + queue: &mut FirmwareQueues, + queues: Arc>, + pool: &mut crate::g16_tvb::Tvb, + owner: Arc>, + scene: u32, + timestamps: [u64; 4], + dependencies: &[Option; 2], + ) -> Result { + if self.render_failed { + return Err(EIO); + } + if !params.valid() || !(2..=3).contains(&queue.priority) { + return Err(EINVAL); + } + let space = client.roots(); + let mut params = *params; + params.tiling_admission = pool.addresses.buffer_manager_slot; + params.fragment_admission = pool.addresses.buffer_manager_slot; + // These accelerator pointers must select the same leased scene as + // BufferThing. Reusing scene 1 here corrupts other in-flight renders. + params.scene_slot = u64::from(scene); + params.cycle = 0x240000 + u64::from(scene) * 0x30; + let scratch = client.prepare_render(&mut params)?; + let mut a = render::Addresses::bootstrap().publication(self.render_publication); + pool.work_addresses(&mut a, scene); + // Keep profiling storage private to the Work. Public timestamp BOs + // are written by the firmware through its separate user destinations. + let lanes = queue.lanes.as_ref().ok_or(EIO)?; + a.tiling_queue = lanes[0].queue; + a.fragment_queue = lanes[1].queue; + a.tiling_event = u64::from(queue.event(0)?); + a.fragment_event = u64::from(queue.event(1)?); + a.support = support.address(); + a.render_shared_state = support.address() + 0x1000; + a.tiling_status_page = params.ta_status & !0x3fff; + a.fragment_status_page = params.fragment_status & !0x3fff; + let queue_ordinal = queue.render_count; + a.fragment_counter = queue_ordinal.wrapping_mul(2); + a.tiling_counter = a.fragment_counter.wrapping_add(1); + // Firmware's dependency graph requires consecutive stamp updates + // within each lane, even when other queues publish between them. + a.fragment_stamp = u64::from(g16_stamp::value(queue_ordinal, 0x100)); + a.tiling_stamp = u64::from(g16_stamp::value(queue_ordinal, 0xc00)); + let va = self.allocate_work(true)?; + a.tiling_shared_tail = va + 0x10000; + a.fragment_shared_tail = va + 0x14000; + a.event_control = va + 0x200; + a.event_count_array = va + 0x300; + // The notifier pass stores driver stamps, then sends the IRQ after + // a DSB. These destinations must be firmware-uncached: a DSB does + // not clean a cached stamp, leaving Linux asleep on stale memory. + a.tiling_driver_stamp = va + 0x18000; + a.fragment_driver_stamp = va + 0x18040; + a.tiling_firmware_stamp = lanes[0].firmware_stamp(); + a.fragment_firmware_stamp = lanes[1].firmware_stamp(); + let alias = 0x72_0000_0000 + (va - 0xffff_fc22_0000_0000); + if alias + 0x8000 > 0x73_0000_0000 { + return Err(ENOSPC); + } + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + for stage in 0..2u64 { + let work_page = va + stage * 0x8000; + let register_page = alias + stage * 0x4000; + let pa = fw.physical(work_page)?; + for view in [self._address_space.as_mut().ok_or(EIO)?, &mut *client] { + view.map_work_alias(register_page, pa)?; + view.sync(); + } + if stage == 0 { + a.tiling_work = work_page + (a.tiling_work & 0x3fff); + a.tiling_microsequence = work_page + 0x4000 + (a.tiling_microsequence & 0x3fff); + a.tiling_prelude = work_page + 0x4000; + a.tiling_register_alias = register_page + (a.tiling_register_alias & 0x3fff); + } else { + a.fragment_work = work_page + (a.fragment_work & 0x3fff); + a.fragment_microsequence = work_page + 0x4000 + (a.fragment_microsequence & 0x3fff); + a.fragment_prelude = work_page + 0x4000; + a.fragment_register_alias = register_page + (a.fragment_register_alias & 0x3fff); + } + } + fw.sync(); + self.invalidate_client(self._address_space.as_ref().ok_or(EIO)?); + self.invalidate_client(client); + // A live context lease keeps these roots fixed through retirement. + a.context_id = u64::from(self.acquire_context(space, 9)?); + a.tiling_user_timestamp_start = timestamps[0]; + a.tiling_user_timestamp_end = timestamps[1]; + a.fragment_user_timestamp_start = timestamps[2]; + a.fragment_user_timestamp_end = timestamps[3]; + // From here any publication failure quarantines the roots. + self.render_failed = true; + let mut ta = params.tiling_work(&a).ok_or(EINVAL)?; + let mut frag = params.fragment_work(&a).ok_or(EINVAL)?; + // Keep firmware's internal profiling destinations distinct from the + // explicit user timestamp slots used for UAPI completion above. + ta[0x8c8..0x8d0].copy_from_slice(&(va + 0x100).to_le_bytes()); + ta[0x8d0..0x8d8].copy_from_slice(&(va + 0x108).to_le_bytes()); + frag[0xc10..0xc18].copy_from_slice(&(va + 0x110).to_le_bytes()); + frag[0xc18..0xc20].copy_from_slice(&(va + 0x118).to_le_bytes()); + // TVB initialization and first publication of a firmware queue are + // independent: several public queues can share one VM's TVB pool. + let initbm = !pool.initialized; + + // The shared count covers individual Work items, not render pairs. + // Firmware retires one tiling and one fragment item per submission. + let submitted = support.submissions.load(Ordering::Relaxed); + let next_submitted = submitted.wrapping_add(2); + // Firmware observes the low word of this cumulative count. + let use_count = next_submitted as u32; + // From here an error conservatively quarantines the client root. + self.render_failed = true; + self._firmware_space + .as_mut() + .ok_or(EIO)? + .write_live(a.render_shared_state, &use_count.to_le_bytes())?; + let (heads, epochs) = + self.publish_render(&a, &ta, &frag, queue, initbm, pool.blocks, dependencies)?; + pool.initialized = true; + support.submissions.store(next_submitted, Ordering::Relaxed); + self.render_publication = self.render_publication.wrapping_add(1); + queue.render_count = queue_ordinal.wrapping_add(1); + let lanes = queue.lanes.as_mut().ok_or(EIO)?; + for stage in 0..2 { + lanes[stage].head = heads[stage]; + lanes[stage].new = false; + } + let stamp = Stamp { + address: a.fragment_firmware_stamp, + value: a.fragment_stamp as u32, + event: a.fragment_event as u32, + }; + let receipt = Receipt { id: va, stamp }; + queue.last[0] = Some(stamp); + queue.pending += 1; + self.flights.push( + Flight { + id: va, + owner: queues, + context: a.context_id as u32, + started: Instant::now(), + stamps: [ + Some(Stamp { + address: a.tiling_firmware_stamp, + value: a.tiling_stamp as u32, + event: a.tiling_event as u32, + }), + Some(stamp), + ], + driver_stamps: [ + Some(Stamp { + address: a.tiling_driver_stamp, + value: a.tiling_stamp as u32, + event: a.tiling_event as u32, + }), + Some(Stamp { + address: a.fragment_driver_stamp, + ..stamp + }), + ], + scratch, + client: owner.clone(), + queues: [(lanes[0].pointers, heads[0]), (lanes[1].pointers, heads[1])], + channels: [ + (queue.priority as usize * 3, (epochs[0] + 1) & 255), + (queue.priority as usize * 3 + 1, (epochs[1] + 1) & 255), + ], + render: Some(RenderFlight { + addresses: a, + client: owner, + root: space.low, + events: RenderEvents::default(), + scene, + }), + }, + GFP_KERNEL, + )?; + self.render_failed = false; + self.batch_works += 1; + self.trace_publication("render", va)?; + self.trace_dependencies("render", va, a.context_id as u32, dependencies)?; + Ok(receipt) + } + + fn publish_render( + &mut self, + a: &render::Addresses, + ta: &[u8], + frag: &[u8], + queue: &FirmwareQueues, + initbm: bool, + blocks: u32, + dependencies: &[Option; 2], + ) -> Result<([u32; 2], [u32; 2])> { + let priority = queue.priority; + let lanes = queue.lanes.as_ref().ok_or(EIO)?; + let publication = self.render_publication; + let firsts = [lanes[0].head, lanes[1].head]; + let heads = [ + (firsts[0] + 1 + u32::from(initbm) + dependencies.iter().flatten().count() as u32) + % 0x500, + (firsts[1] + 2) % 0x500, + ]; + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + fw.write_live(a.event_control, &a.event_count_array.to_le_bytes())?; + fw.write_live( + a.event_control + 8, + &(a.fragment_stamp as u32).to_le_bytes(), + )?; + fw.write_live(a.event_control + 12, &0u32.to_le_bytes())?; + fw.write_live(a.event_control + 0x10, &0x50u32.to_le_bytes())?; + fw.write_live(a.event_control + 0x24, &(a.context_id as u32).to_le_bytes())?; + fw.write_live(a.event_count_array, &2u32.to_le_bytes())?; + fw.write_live( + a.tiling_driver_stamp, + &g16_stamp::previous(a.tiling_stamp as u32).to_le_bytes(), + )?; + fw.write_live( + a.fragment_driver_stamp, + &g16_stamp::previous(a.fragment_stamp as u32).to_le_bytes(), + )?; + // The queue lanes persist independently of the physical work channel. + for stage in 0..2 { + let Lane { ring, .. } = lanes[stage]; + let prelude = if stage == 0 { + a.tiling_prelude + } else { + a.fragment_prelude + }; + let work = if stage == 0 { + a.tiling_work + } else { + a.fragment_work + }; + let micro = if stage == 0 { + a.tiling_microsequence + } else { + a.fragment_microsequence + }; + fw.write_live( + prelude, + &if stage == 0 { + a.tiling_prelude(blocks) + } else { + a.fragment_prelude() + }, + )?; + fw.write_live(work, if stage == 0 { ta } else { frag })?; + if stage == 0 { + fw.write_live(micro, &a.tiling_microsequence())?; + } else { + fw.write_live(micro, &a.fragment_microsequence())?; + } + let mut pos = firsts[stage]; + if stage == 0 { + // The prelude page leaves +0x40..+0xc0 free before its + // microsequence. All barrier bytes share the Work's lifetime. + for (index, dep) in dependencies.iter().flatten().enumerate() { + let address = prelude + 0x40 + index as u64 * 0x40; + fw.write_live( + address, + &g16_fw::queue::dependency( + dep.address, + dep.value, + dep.event, + a.tiling_stamp as u32, + a.tiling_uuid as u32, + ), + )?; + fw.write_live(ring + u64::from(pos) * 8, &address.to_le_bytes())?; + pos = (pos + 1) % 0x500; + } + } + if stage == 1 || initbm { + fw.write_live(ring + u64::from(pos) * 8, &prelude.to_le_bytes())?; + pos = (pos + 1) % 0x500; + } + fw.write_live(ring + u64::from(pos) * 8, &work.to_le_bytes())?; + } + let mut epochs = [0; 2]; + for stage in [1usize, 0] { + let mut lane = lanes[stage]; + lane.head = heads[stage]; + epochs[stage] = self.stage_queue( + priority as usize * 3 + stage, + lane, + if stage == 0 { + a.tiling_event + } else { + a.fragment_event + } as u32, + publication.wrapping_add(1), + )?; + } + Ok((heads, epochs)) + } +} + +impl Bootstrap { + pub(crate) fn map_compute_work( + &mut self, + space: &mut g16_vm::AddressSpace, + private: core::ops::Range, + ) -> Result { + let opening = self._opening_client.as_mut().ok_or(EIO)?; + for (base, size) in compute::private_ranges() { + // Scratch and marker are private to each command at fresh VAs. + // Only the device-owned FList directory and spill pool are shared. + if base == compute::SCRATCH || base == compute::MARKER { + continue; + } + for va in (base..base + size as u64).step_by(0x4000) { + if space.low.translate(va)?.is_some() { + return Err(EEXIST); + } + let pa = opening.low.translate(va)?.ok_or(EIO)?; + space.low.map_pages( + va..va + 0x4000, + pa, + crate::pgtable::prot::PROT_GPU_SHARED_RW, + false, + )?; + } + } + space.init_compute_private(private); + space.sync(); + Ok(()) + } + + pub(crate) fn submit_compute( + &mut self, + queues: Arc>, + owner: Arc>, + params: &compute::Parameters, + timestamps: [u64; 4], + dependencies: &[Option; 2], + ) -> Result { + let mut queue = queues.lock(); + let mut client = owner.lock(); + self.reserve_flight( + &queue, + false, + client.roots().low, + [0, 0, 1 + dependencies.iter().flatten().count() as u32], + )?; + self.prepare_queues(&mut queue)?; + let result = self.submit_compute_queue( + &mut queue, + queues.clone(), + &mut client, + owner.clone(), + params, + timestamps, + dependencies, + ); + if queue.pending == 0 { + queue.events = None; + } + result + } + + fn submit_compute_queue( + &mut self, + queue: &mut FirmwareQueues, + queues: Arc>, + client: &mut g16_vm::AddressSpace, + owner: Arc>, + params: &compute::Parameters, + timestamps: [u64; 4], + dependencies: &[Option; 2], + ) -> Result { + let lane = queue.lanes.as_ref().ok_or(EIO)?[2]; + let priority = queue.priority; + + let mut params = *params; + let scratch = client.prepare_compute(&mut params)?; + let space = client.roots(); + if self.render_failed { + return Err(EIO); + } + if !(2..=3).contains(&priority) { + return Err(EINVAL); + } + let ordinal = queue.compute_count; + let work = self.allocate_work(false)?; + let alias = 0x71_0000_0000 + (work - 0xffff_fc22_0000_0000); + if alias + 0x4000 > 0x72_0000_0000 { + return Err(ENOSPC); + } + // Firmware save state is independent of shader resource tables. + // The upper half of the zeroed Work page is GPU-addressable storage. + params.save_area = alias + 0x2000; + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + let pa = fw.physical(work)?; + self._address_space + .as_mut() + .ok_or(EIO)? + .map_work_alias(alias, pa)?; + client.map_work_alias(alias, pa)?; + fw.zero_live(work + 0x2000, 0x2000)?; + let mut a = compute::Addresses::publication(self.compute_publication, ordinal, work, alias); + a.queue = lane.queue; + a.driver_stamp = work + 0x4000; + a.firmware_stamp = lane.firmware_stamp(); + a.event = queue.event(2)?; + self._address_space.as_ref().ok_or(EIO)?.sync(); + fw.sync(); + client.sync(); + self.invalidate_client(self._address_space.as_ref().ok_or(EIO)?); + self.invalidate_client(client); + a.context = self.acquire_context(space, 1)?; + a.identity = (u64::from(a.context) << 32) | (a.identity & 0xffff_ffff); + a.register_identity = (u64::from(a.context) << 32) | 1; + self.render_failed = true; + a.cdm_entry = Some(alias + compute::ENTRY_OFFSET); + let work = params.work(&a).ok_or(EINVAL)?; + // Timestamp's user destinations must lie in the validated special- + // object aperture. Firmware writes nanoseconds directly, as on M1/M2. + // The pointer pair is private to this Work and may be consumed/reset. + let user_timestamps = a.work + 0x17c0; + let count = self.compute_publication.wrapping_add(1); + let count32 = count as u32; + + let first = lane.head; + let head = (first + 1 + dependencies.iter().flatten().count() as u32) % 0x500; + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + fw.write_live(compute::SHARED_STATE, &count32.to_le_bytes())?; + fw.write_live(a.work, &work)?; + fw.write_live(a.work + compute::ENTRY_OFFSET, &compute::entry(params.cdm))?; + let mut microsequence = params.microsequence(&a); + for base in [0x1bc, 0x20c] { + microsequence[base + 0x24..base + 0x2c].copy_from_slice(&user_timestamps.to_le_bytes()); + } + fw.write_live(user_timestamps, ×tamps[0].to_le_bytes())?; + fw.write_live(user_timestamps + 8, ×tamps[1].to_le_bytes())?; + fw.write_live(a.microsequence, µsequence)?; + let notifier = a.notifier(0); + // Retained notifier list links are firmware-owned even after completion. + fw.write_live(a.notifier, ¬ifier[..0x14])?; + fw.write_live(a.notifier + 0x24, ¬ifier[0x24..0x28])?; + fw.write_live(a.threshold, &1u32.to_le_bytes())?; + fw.write_live(a.driver_stamp, &g16_stamp::previous(a.stamp).to_le_bytes())?; + let mut pos = first; + for (index, dep) in dependencies.iter().flatten().enumerate() { + // Between the threshold/timestamps and the CDM entry trampoline. + let address = a.work + 0x1900 + index as u64 * 0x40; + fw.write_live( + address, + &g16_fw::queue::dependency( + dep.address, + dep.value, + dep.event, + a.stamp, + a.identity as u32, + ), + )?; + fw.write_live(lane.ring + u64::from(pos) * 8, &address.to_le_bytes())?; + pos = (pos + 1) % 0x500; + } + fw.write_live(lane.ring + u64::from(pos) * 8, &a.work.to_le_bytes())?; + let channel = priority as usize * 3 + 2; + let mut pending_lane = lane; + pending_lane.head = head; + let epoch = + self.stage_queue(channel, pending_lane, a.event, count)?; + self.compute_publication = count; + queue.compute_count = ordinal.wrapping_add(1); + let lane = &mut queue.lanes.as_mut().ok_or(EIO)?[2]; + lane.head = head; + lane.new = false; + let stamp = Stamp { + address: a.firmware_stamp, + value: a.stamp, + event: a.event, + }; + let pointers = lane.pointers; + queue.last[1] = Some(stamp); + queue.pending += 1; + self.flights.push( + Flight { + id: a.work, + owner: queues.clone(), + context: a.context, + started: Instant::now(), + stamps: [Some(stamp), None], + driver_stamps: [ + Some(Stamp { + address: a.driver_stamp, + ..stamp + }), + None, + ], + scratch, + client: owner, + queues: [(pointers, head), (0, 0)], + channels: [(channel, (epoch + 1) & 255), (usize::MAX, 0)], + render: None, + }, + GFP_KERNEL, + )?; + self.render_failed = false; + self.batch_works += 1; + self.trace_publication("compute", a.work)?; + self.trace_dependencies("compute", a.work, a.context, dependencies)?; + Ok(Receipt { id: a.work, stamp }) + } +} + +#[derive(Default)] +struct RenderEvents { + memory_limit: bool, +} + +impl Bootstrap { + /// Consume only events owned by this publication. Growth transfers retained + /// pages -> block list -> host counts -> reply -> producer, as in the shim. + fn service_render_events( + &mut self, + client: &mut g16_vm::AddressSpace, + pool: &mut crate::g16_tvb::Tvb, + a: &render::Addresses, + service: &mut RenderEvents, + ) -> Result { + const STATE: u64 = 0xfffffc200003f1c0; + const RING: u64 = 0xfffffc200003f440; + let fw = self._firmware_space.as_mut().ok_or(EIO)?; + let mut cursor = fw.read_u32(STATE)?; + let end = fw.read_u32(STATE + 0x20)?; + if cursor >= 256 || end >= 256 { + return Err(EIO); + } + while cursor != end { + let base = RING + u64::from(cursor) * 0x48; + let kind = fw.read_u32(base)?; + let next = (cursor + 1) & 255; + if kind == 1 { + fw.write_live(STATE, &next.to_le_bytes())?; + cursor = next; + continue; + } + if !matches!(kind, 6 | 7) { + dev_err!(self.dev.as_ref(), "G16: unexpected render Event {}\n", kind); + return Err(EIO); + } + let guard = self._sgx.try_access().ok_or(ENODEV)?; + let sgx = &*guard; + let head = sgx.try_read32(0xd640c0)?; + let ack = sgx.try_read32(0xd640c8)?; + let tail = sgx.try_read32(0xd60060)?; + drop(guard); + if head >= 256 || ack >= 256 || tail >= 256 { + return Err(EIO); + } + let successor = (tail + 1) & 255; + if successor == head || successor == ack { + break; + } + let mut reply = [0u8; 64]; + if kind == 6 { + let subpipe = fw.read_u32(base + 16)?; + let halt_count = fw.read_u64(base + 20)?; + if fw.read_u32(base + 4)? != a.context_id as u32 + || fw.read_u32(base + 8)? != pool.addresses.buffer_manager_slot as u32 + || fw.read_u32(base + 12)? != pool.counter + || subpipe >= 64 + { + return Err(EIO); + } + let grown = pool.grow(fw, client, a.context_id as u8)?; + dev_info!( + self.dev.as_ref(), + "G16: TVB slot {} blocks {} growth {} subpipe {}\n", + pool.addresses.buffer_manager_slot, + pool.blocks, + grown, + subpipe + ); + for (i, word) in [ + 8, + u32::from(grown), + pool.addresses.buffer_manager_slot as u32, + a.context_id as u32, + pool.counter, + ] + .iter() + .enumerate() + { + reply[i * 4..i * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + // Native allocateMemoryEvent echoes the request's subpipe + // and halt counter. Queued work can request growth on another + // subpipe before its main stage starts, so this state belongs + // to the pool, not whichever Work is currently executing. + reply[20..24].copy_from_slice(&subpipe.to_le_bytes()); + reply[24..32].copy_from_slice(&halt_count.to_le_bytes()); + pool.refused |= !grown; + pool.counter = pool.counter.checked_add(1).ok_or(EIO)?; + } else { + if !pool.refused + || service.memory_limit + || fw.read_u32(base + 4)? != 0 + || fw.read_u32(base + 8)? != 1 + || fw.read_u32(base + 12)? != a.fragment_event as u32 + || fw.read_u32(base + 16)? != a.fragment_stamp as u32 + { + return Err(EIO); + } + reply[..4].copy_from_slice(&9u32.to_le_bytes()); + reply[4..8].copy_from_slice(&1u32.to_le_bytes()); + reply[8..16].copy_from_slice(&a.fragment_queue.to_le_bytes()); + reply[16..20].copy_from_slice(&(a.fragment_stamp as u32).to_le_bytes()); + service.memory_limit = true; + } + crate::mem::sync(); + let control = g16_fw::MainConfig::bootstrap(g16_fw::BUNDLE_ADDRESS).channels[12].ring; + fw.write_live(control + u64::from(tail) * 64, &reply)?; + crate::mem::sync(); + fw.write_live(STATE, &next.to_le_bytes())?; + let guard = self._sgx.try_access().ok_or(ENODEV)?; + guard.try_write32(successor, 0xd60060)?; + drop(guard); + crate::mem::sync(); + self._rtkit + .as_mut() + .send_message(0x21, 0x0084_0000_0000_0011)?; + return Ok(()); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g16_stamp.rs b/drivers/gpu/drm/asahi/g16_stamp.rs new file mode 100644 index 0000000000000000000000000000000000000000..a06650e58f6eaca8d4cbf6c6c9d46def27fe8a7d --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_stamp.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Wrapping firmware completion stamps, independent of software sequence counts. + +const STEP: u32 = 0x100; + +/// The first command has ordinal zero. Only the firmware stamp wraps; callers +/// retain the full software ordinal for accounting and command identities. +pub(crate) fn value(ordinal: u64, first: u32) -> u32 { + first.wrapping_add((ordinal as u32).wrapping_mul(STEP)) +} + +/// Initialize a private completion destination to an uncompleted value. Zero +/// cannot serve as a sentinel: it is a valid stamp and sorts after high stamps. +pub(crate) fn previous(target: u32) -> u32 { + target.wrapping_sub(STEP) +} + +/// Queue admission bounds outstanding work to much less than half the stamp +/// range, making this signed modular comparison unambiguous across rollover. +pub(crate) fn reached(current: u32, target: u32) -> bool { + (current.wrapping_sub(target) as i32) >= 0 +} diff --git a/drivers/gpu/drm/asahi/g16_tvb.rs b/drivers/gpu/drm/asahi/g16_tvb.rs new file mode 100644 index 0000000000000000000000000000000000000000..c17efe221669a8020e3ae91309f594ba5eb54c61 --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_tvb.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors + +//! Context-owned TVB lists and retained growth backing, following g16g_tvb.py. +use crate::{ + g16_fw, g16_render as render, + g16_vm::{AddressSpace, FirmwareSpace}, + pgtable::prot, +}; +use kernel::prelude::*; + +pub(crate) const GROW_BLOCKS: u32 = 10; +pub(crate) const MAX_BLOCKS: u32 = 11 + 32 * GROW_BLOCKS; + +pub(crate) struct Tvb { + pub(crate) root: u64, + pub(crate) addresses: render::Addresses, + pub(crate) initialized: bool, + pub(crate) blocks: u32, + pub(crate) counter: u32, + pub(crate) refused: bool, + scenes: u64, + next: u64, +} +impl Tvb { + pub(crate) fn new( + fw: &mut FirmwareSpace, + client: &mut AddressSpace, + base: u64, + slot: u64, + growth_base: u64, + ) -> Result { + let mut a = render::Addresses::bootstrap(); + a.buffer_manager_slot = slot; + a.buffer_manager = base; + a.buffer_thing = base + 0x4080; + a.buffer_manager_block_control = base + 0x8000; + a.buffer_manager_scene_list = base + 0xc000; + a.buffer_manager_counter = base + 0x10000; + a.buffer_manager_block_list = base + 0x14000; + a.buffer_manager_page_list = base + 0x24000; + fw.alloc(base, 0x40000, prot::PROT_FW_PRIV_RW)?; + for va in (base..base + 0x40000).step_by(0x4000) { + fw.init_page(va, |bytes| a.private_page(va, bytes).ok_or(EINVAL))?; + } + // Firmware and the accelerator must mutate the same page-list backing. + // The retired packed initialization pages remain owned by this context. + for offset in (0..0x1c000u64).step_by(0x4000) { + let va = 0x1000258000 + offset; + let pa = fw.physical(a.buffer_manager_page_list + offset)?; + client.low.unmap_pages(va..va + 0x4000)?; + client + .low + .map_pages(va..va + 0x4000, pa, prot::PROT_GPU_SHARED_RW, false)?; + } + fw.write_live( + 0xfffffc2000128000 + slot * 16, + &g16_fw::parameter_buffer_descriptor(0x1000258000, 44).ok_or(EINVAL)?, + )?; + fw.sync(); + client.sync(); + Ok(Self { + root: client.roots().low, + addresses: a, + initialized: false, + blocks: 11, + counter: 0, + refused: false, + scenes: 0, + next: growth_base, + }) + } + + pub(crate) fn reserve_scene(&mut self) -> Result { + // The firmware's accelerator scene records have 36 slots. Lease one + // until both stages and their event replies have retired. + let index = (!self.scenes).trailing_zeros(); + if index >= 36 { + return Err(EBUSY); + } + self.scenes |= 1 << index; + Ok(index) + } + + pub(crate) fn release_scene(&mut self, index: u32) { + self.scenes &= !(1 << index); + } + + pub(crate) fn work_addresses(&self, a: &mut render::Addresses, scene: u32) { + let p = &self.addresses; + a.buffer_manager_slot = p.buffer_manager_slot; + a.buffer_manager = p.buffer_manager; + a.buffer_manager_block_control = p.buffer_manager_block_control; + a.buffer_manager_scene_list = p.buffer_manager_scene_list; + a.buffer_manager_counter = p.buffer_manager_counter; + a.buffer_manager_block_list = p.buffer_manager_block_list; + a.buffer_manager_page_list = p.buffer_manager_page_list; + a.buffer_thing = (p.buffer_thing & !0x3fff) + u64::from(scene) * 0x80; + } + + pub(crate) fn grow( + &mut self, + fw: &mut FirmwareSpace, + client: &mut AddressSpace, + asid: u8, + ) -> Result { + let a = &self.addresses; + let old = self.blocks; + if client.roots().low != self.root + // Another queued request can arrive before firmware consumes the + // previous growth reply. Its mirror may lag our published lists. + || fw.read_u32(a.buffer_manager + 0x3c)? > old + || fw.read_u32(a.buffer_manager_block_control)? != old + || fw.read_u32(a.buffer_manager_block_control + 4)? != old + || fw.read_u64(a.buffer_manager + 0x44)? != a.buffer_manager_block_list + || fw.read_u64(a.buffer_manager + 0x4c)? != a.buffer_manager_block_control + || fw.read_u32(a.buffer_manager + 0x58)? != 0x20000 + { + return Err(EIO); + } + let limit = (*crate::module_parameters::tvb_max_blocks.value()).min(MAX_BLOCKS); + let new = (old + GROW_BLOCKS).min(limit).max(old); + if new > fw.read_u32(a.buffer_manager + 0x38)? || new * 8 > 0x10000 || new * 16 > 0x1c000 { + return Err(EIO); + } + if new == old { + return Ok(false); + } + // Allocation is all-or-nothing before publishing any leaf or list entry. + // After mapping starts, any error requires retaining the context/reset. + if !client.alloc_tvb_blocks(self.next, (new - old) as usize)? { + return Ok(false); + } + client.sync(); + client.low.invalidate(Some(asid)); + client.high.invalidate(Some(asid)); + crate::mem::sync(); + client.low.clear_invalidations(); + client.high.clear_invalidations(); + let mut entries = [0u8; GROW_BLOCKS as usize * 8]; + for index in 0..new - old { + let dva = self.next + u64::from(index) * 0x28000; + let id = (dva - render::CONTEXT_BASE) / 0x8000; + let offset = index as usize * 8; + entries[offset..offset + 8].copy_from_slice(&id.to_le_bytes()); + } + fw.write_live( + a.buffer_manager_block_list + u64::from(old) * 8, + &entries[..(new - old) as usize * 8], + )?; + crate::mem::sync(); + let counts = (u64::from(new) | (u64::from(new) << 32)).to_le_bytes(); + fw.write_live(a.buffer_manager_block_control, &counts)?; + crate::mem::sync(); + // Firmware expands its page list and counts after the type-8 reply. + self.blocks = new; + self.next += u64::from(new - old) * 0x28000; + Ok(true) + } +} diff --git a/drivers/gpu/drm/asahi/g16_vm.rs b/drivers/gpu/drm/asahi/g16_vm.rs new file mode 100644 index 0000000000000000000000000000000000000000..4bbec51432a18c83ff12f16769e0dfb3278344ea --- /dev/null +++ b/drivers/gpu/drm/asahi/g16_vm.rs @@ -0,0 +1,836 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from Niklas Sheth's linux-m4-integration prototype. + +//! G16 client roots. The firmware bootstrap root is separate from these +//! roots, even when the hardware context table is using slot zero. + +use crate::pgtable::{prot, UatPageTable, UAT_PGSZ}; +use core::cell::{Cell, RefCell}; +use kernel::rbtree::{RBTree, RBTreeNode}; +use kernel::{device, page::Page, prelude::*}; + +const IAS: usize = 42; +const OAS: u32 = 42; +const USC_BASE: u64 = 0x100_0000_0000; + +/// Published root addresses. The submission owner must retain their VM until +/// GPU completion; copying these addresses does not transfer page ownership. +#[derive(Clone, Copy)] +pub(crate) struct Roots { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) struct AddressSpace { + pub(crate) low: UatPageTable, + pub(crate) high: UatPageTable, + // Drop the owned tables before releasing their mapped pages. + pages: RBTree>, + compute_scratch: core::ops::Range, + render_scratch: core::ops::Range, + scratch_free: [KVec>; 2], + scratch_users: [usize; 2], +} + +/// An exclusive suballocation; its backing remains owned by the VM. +#[derive(Debug, PartialEq)] +pub(crate) struct Scratch { + range: core::ops::Range, + render: bool, +} + +impl AddressSpace { + /// Reused Work retains the same alias and backing. Existing aliases need + /// no PTE edit or invalidation; conflicting client mappings are rejected. + pub(crate) fn map_work_alias(&mut self, va: u64, pa: u64) -> Result { + match self.low.translate(va)? { + Some(existing) if existing == pa => Ok(()), + Some(_) => Err(EEXIST), + None => self.low.map_pages( + va..va + UAT_PGSZ as u64, + pa, + prot::PROT_GPU_SHARED_RW, + false, + ), + } + } + pub(crate) fn roots(&self) -> Roots { + Roots { + low: self.low.ttb(), + high: self.high.ttb(), + } + } + + pub(crate) fn new() -> Result { + Ok(Self { + low: UatPageTable::new_with_ias(OAS, IAS)?, + high: UatPageTable::new_with_ias(OAS, IAS)?, + pages: RBTree::new(), + compute_scratch: 0..0, + render_scratch: 0..0, + scratch_free: [KVec::new(), KVec::new()], + scratch_users: [0; 2], + }) + } + + pub(crate) fn alloc_low( + &mut self, + va: u64, + size: usize, + access: crate::pgtable::Prot, + ) -> Result { + if size == 0 || (va as usize | size) & (UAT_PGSZ - 1) != 0 { + return Err(EINVAL); + } + let end = va.checked_add(size as u64).ok_or(EINVAL)?; + for v in (va..end).step_by(UAT_PGSZ) { + if self.pages.get(&v).is_some() || self.low.translate(v)?.is_some() { + return Err(EEXIST); + } + } + for v in (va..end).step_by(UAT_PGSZ) { + let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + // These raw pages do not receive the GEM/DMA mapping's initial + // publication. Clean CPU initialization before exposing them. + clean_page(&page); + let phys = page.phys(); + self.pages.try_create_and_insert(v, page, GFP_KERNEL)?; + self.low + .map_pages(v..v + UAT_PGSZ as u64, phys, access, false)?; + } + Ok(()) + } + + /// Build backend-only render storage before this root is published. + /// Allocation failures leave any acquired pages owned by this address space; + /// callers discard the unpublished VM rather than reusing a partial graph. + pub(crate) fn alloc_render_private(&mut self, start: u64, end: u64) -> Result { + use crate::g16_render as render; + let mut params = render::Parameters::default(); + params.set_private(start, end).ok_or(EINVAL)?; + let pool = render::OperandPool::new(start, end).ok_or(EINVAL)?; + self.render_scratch = + pool.growth_end().ok_or(EINVAL)?..end.min(render::CONTEXT_BASE + (1 << 32)); + if self.render_scratch.start > self.render_scratch.end { + return Err(ENOMEM); + } + self.alloc_low(start, render::PRIVATE_SIZE, prot::PROT_GPU_SHARED_RW)?; + for offset in (0..render::PRIVATE_SIZE).step_by(UAT_PGSZ) { + let va = start + offset as u64; + let page = self.pages.get(&va).ok_or(EIO)?; + page.with_pointer_into_page(0, UAT_PGSZ, |ptr| { + // SAFETY: The closure has exclusive access to the full page + // owned by this unpublished VM; no GPU root references it. + let bytes = unsafe { core::slice::from_raw_parts_mut(ptr, UAT_PGSZ) }; + render::private_page(&pool, offset, bytes).ok_or(EINVAL) + })?; + clean_page(page); + } + for (alias, offset, size) in render::private_aliases(&pool) { + for page in (0..size).step_by(UAT_PGSZ) { + let va = alias + page as u64; + if self.low.translate(va)?.is_some() { + return Err(EEXIST); + } + let physical = self + .low + .translate(start + offset + page as u64)? + .ok_or(EIO)?; + self.low.map_pages( + va..va + UAT_PGSZ as u64, + physical, + prot::PROT_GPU_SHARED_RW, + false, + )?; + } + } + self.sync(); + Ok(()) + } + + /// Each active render exclusively leases tile metadata and save storage. + pub(crate) fn prepare_render( + &mut self, + p: &mut crate::g16_render::Parameters, + ) -> Result { + let (tilemap, tpc) = p.scratch_sizes().ok_or(EINVAL)?; + let size = tilemap + .checked_add(tpc) + .and_then(|n| n.checked_add(5 * UAT_PGSZ)) + .ok_or(ENOMEM)?; + let scratch = self.lease_scratch(true, size)?; + let base = scratch.range.start; + p.tilemap = base; + p.tpc = base + tilemap as u64; + let meta = p.tpc + tpc as u64; + p.layermeta = meta; + p.heapmeta = meta + if p.layers > 1 { 0x100 } else { 0 }; + p.deflake_3 = meta + 0x4000; + p.deflake_2 = p.deflake_3 + 0x20; + p.deflake_1 = p.deflake_3 + 0x2a0; + p.ta_status = meta + 0x8000 + 0x240; + p.fragment_status = meta + 0xc000 + 0x2c0; + p.aux_fb = meta + 0x10000; + self.write_low(p.aux_fb + 0x600, &0x0000035b60000000u64.to_le_bytes())?; + self.sync(); + Ok(scratch) + } + + /// Compute commands use distinct private scratch addresses in the caller's + /// reserved kernel aperture. The persistent root and all backing remain + /// owned by the VM; allocating a later command never replaces a live PTE. + pub(crate) fn init_compute_private(&mut self, range: core::ops::Range) { + self.compute_scratch = range; + } + + pub(crate) fn prepare_compute( + &mut self, + p: &mut crate::g16_compute::Parameters, + ) -> Result { + let scratch = self.lease_scratch(false, 0x24000)?; + let base = scratch.range.start; + p.scratch = base; + p.marker = base + 0x20000; + Ok(scratch) + } + + fn lease_scratch(&mut self, render: bool, size: usize) -> Result { + if size == 0 || size & (UAT_PGSZ - 1) != 0 { + return Err(EINVAL); + } + let index = usize::from(render); + // Splitting a free range adds at most one future free-list entry. + // Reserve now; completion and coalescing cannot allocate. + self.scratch_free[index].reserve(self.scratch_users[index] + 1, GFP_KERNEL)?; + let free = self.scratch_free[index] + .iter() + .position(|r| r.end - r.start >= size as u64); + let range = if let Some(slot) = free { + let start = self.scratch_free[index][slot].start; + let end = start + size as u64; + if end == self.scratch_free[index][slot].end { + self.scratch_free[index].remove(slot).map_err(|_| EIO)?; + } else { + self.scratch_free[index][slot].start = end; + } + // All previous GPU writes have retired. Reinitialize the entire + // leased range without touching PTEs or neighbouring active jobs. + for va in (start..end).step_by(UAT_PGSZ) { + let page = self.pages.get(&va).ok_or(EIO)?; + page.with_page_mapped(|ptr| { + // SAFETY: The lease owns this full page exclusively. + unsafe { core::ptr::write_bytes(ptr, 0, UAT_PGSZ) }; + }); + clean_page(page); + } + start..end + } else { + let arena = if render { + &mut self.render_scratch + } else { + &mut self.compute_scratch + }; + let start = arena.start; + let end = start.checked_add(size as u64).ok_or(ENOMEM)?; + if end > arena.end { + return Err(ENOMEM); + } + // Partially allocated failures stay quarantined until VM teardown. + arena.start = end; + self.alloc_low(start, size, prot::PROT_GPU_SHARED_RW)?; + start..end + }; + self.scratch_users[index] += 1; + Ok(Scratch { range, render }) + } + + pub(crate) fn release_scratch(&mut self, scratch: Scratch) -> Result { + let index = usize::from(scratch.render); + let mut range = scratch.range; + let free = &mut self.scratch_free[index]; + let mut slot = 0; + while slot < free.len() { + if free[slot].end == range.start || range.end == free[slot].start { + let old = free.remove(slot).map_err(|_| EIO)?; + range.start = range.start.min(old.start); + range.end = range.end.max(old.end); + slot = 0; + } else { + slot += 1; + } + } + free.push(range, GFP_KERNEL)?; + self.scratch_users[index] -= 1; + Ok(()) + } + + /// Allocate all ten-block growth pages before changing visible mappings. + pub(crate) fn alloc_tvb_blocks(&mut self, base: u64, count: usize) -> Result { + if count == 0 || count > 10 || base & 0x7fff != 0 { + return Err(EINVAL); + } + let mut pages = KVec::new(); + if pages.reserve(count * 8, GFP_KERNEL).is_err() { + return Ok(false); + } + for block in 0..count { + let start = base + block as u64 * 0x28000; + for va in (start..start + 0x28000).step_by(UAT_PGSZ) { + if self.pages.get(&va).is_some() || self.low.translate(va)?.is_some() { + return Err(EEXIST); + } + } + for va in (start..start + 0x20000).step_by(UAT_PGSZ) { + let page = match Page::alloc_page(GFP_KERNEL | __GFP_ZERO) { + Ok(page) => page, + Err(_) => return Ok(false), + }; + clean_page(&page); + let node = match RBTreeNode::new(va, page, GFP_KERNEL) { + Ok(node) => node, + Err(_) => return Ok(false), + }; + pages.push(node, GFP_KERNEL)?; + } + } + // Transfer every preallocated node before the first fallible PTE edit. + for node in pages { + self.pages.insert(node); + } + for block in 0..count { + let start = base + block as u64 * 0x28000; + for va in (start..start + 0x20000).step_by(UAT_PGSZ) { + self.low.map_pages( + va..va + UAT_PGSZ as u64, + self.pages.get(&va).ok_or(EIO)?.phys(), + prot::PROT_GPU_SHARED_RW, + false, + )?; + } + } + Ok(true) + } + + pub(crate) fn write_low(&mut self, mut va: u64, mut bytes: &[u8]) -> Result { + while !bytes.is_empty() { + let base = va & !(UAT_PGSZ as u64 - 1); + let page = self.pages.get(&base).ok_or(EINVAL)?; + let offset = (va - base) as usize; + let size = bytes.len().min(UAT_PGSZ - offset); + page.with_pointer_into_page(offset, size, |p| { + // SAFETY: This range belongs to an unpublished client allocation. + unsafe { + core::ptr::copy_nonoverlapping(bytes.as_ptr(), p, size); + } + Ok(()) + })?; + clean_page(page); + va = va.checked_add(size as u64).ok_or(EINVAL)?; + bytes = &bytes[size..]; + } + Ok(()) + } + + /// Publish page-table edits. Owned client pages are cleaned when initialized; + /// mapping changes do not need to sweep all existing client allocations. + /// Firmware-private storage is managed separately by FirmwareSpace. + pub(crate) fn sync(&self) { + self.low.clean(); + self.high.clean(); + crate::mem::sync(); + } + + /// Exercise the ownership pattern needed for the firmware bootstrap + /// root using private pages before touching firmware-owned tables. + pub(crate) fn check_borrowed_root(&mut self, dev: &device::Device) -> Result { + let a = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let b = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let page = UAT_PGSZ as u64; + let far = 1u64 << 36; + self.low + .map_pages(page..2 * page, a.phys(), prot::PROT_GPU_SHARED_RW, false)?; + let result = (|| { + let mut borrowed = + UatPageTable::new_with_ttb_and_ias(self.low.ttb(), 0..1u64 << IAS, OAS, IAS)?; + // Extend both an existing leaf table and a previously empty L1 slot. + borrowed.map_pages( + 2 * page..3 * page, + b.phys(), + prot::PROT_GPU_SHARED_RW, + false, + )?; + borrowed.map_pages(far..far + page, b.phys(), prot::PROT_GPU_SHARED_RW, false)?; + let valid = borrowed.translate(page)? == Some(a.phys()) + && borrowed.translate(2 * page)? == Some(b.phys()) + && borrowed.translate(far)? == Some(b.phys()); + borrowed.unmap_pages(2 * page..3 * page)?; + borrowed.unmap_pages(far..far + page)?; + drop(borrowed); + if !valid + || self.low.translate(page)? != Some(a.phys()) + || self.low.translate(far)?.is_some() + || self.low.translate(2 * page)?.is_some() + { + return Err(EIO); + } + Ok(()) + })(); + self.low.unmap_pages(page..2 * page)?; + result?; + dev_info!( + dev, + "G16: borrowed-root teardown preserved existing mappings\n" + ); + Ok(()) + } + + /// Bring-up check before either root is made visible to firmware. A + /// legacy 39-bit mask aliases the two addresses exercised here, which + /// would silently corrupt client data when installing the USC archive. + pub(crate) fn check_geometry(&mut self, dev: &device::Device) -> Result { + let a = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let b = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let low = UAT_PGSZ as u64; + let usc = USC_BASE + low; + self.low + .map_pages(low..low + low, a.phys(), prot::PROT_GPU_SHARED_RW, false)?; + if let Err(e) = + self.low + .map_pages(usc..usc + low, b.phys(), prot::PROT_GPU_SHARED_RW, false) + { + self.low.unmap_pages(low..low + low)?; + return Err(e); + } + let result = (|| { + if self.low.translate(low + 37)? != Some(a.phys() + 37) + || self.low.translate(usc + 91)? != Some(b.phys() + 91) + || self.high.translate(usc)?.is_some() + { + return Err(EIO); + } + self.low.unmap_pages(usc..usc + low)?; + if self.low.translate(usc)?.is_some() || self.low.translate(low)? != Some(a.phys()) { + return Err(EIO); + } + Ok(()) + })(); + // Remove both mappings on all paths before their backing pages drop. + // The upper mapping has already been removed on the successful path. + if self.low.translate(usc)?.is_some() { + self.low.unmap_pages(usc..usc + low)?; + } + self.low.unmap_pages(low..low + low)?; + result?; + dev_info!( + dev, + "G16: 42-bit UAT mapping/unmapping verified across USC base; roots {:#x}/{:#x}\n", + self.low.ttb(), + self.high.ttb() + ); + Ok(()) + } +} + +use kernel::types::Owned; + +struct FirmwareRegion { + va: u64, + pages: KVec>, + // One flag per owned page. Live writes publish themselves; only allocation + // and unpublished initialization need work at the next space-wide sync. + dirty_pages: KVec>, + pending: Cell, +} + +/// The firmware uses this bootloader root independently of all client slots. +/// Its existing tables remain owned by the firmware reservation. +pub(crate) struct FirmwareSpace { + table: UatPageTable, + regions: RBTree, + dirty_regions: RefCell>, + external: KVec>, +} + +impl FirmwareSpace { + pub(crate) fn attach(dev: &device::Device) -> Result { + let node = dev.of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(kernel::c_str!("pagetables"))?; + let table = UatPageTable::new_with_ttb_and_ias( + res.start(), + 0xffff_fc00_0000_0000..0xffff_ffff_ffff_c000, + OAS, + IAS, + )?; + dev_info!(dev, "G16: attached firmware root {:#x}\n", table.ttb()); + Ok(Self { + table, + regions: RBTree::new(), + dirty_regions: RefCell::new(KVec::new()), + external: KVec::new(), + }) + } + + /// Borrow a hardware aperture. No ownership of physical pages is implied. + pub(crate) fn map_external( + &mut self, + va: u64, + pa: u64, + size: usize, + prot: crate::pgtable::Prot, + ) -> Result { + if size == 0 || (va | pa | size as u64) & (UAT_PGSZ as u64 - 1) != 0 { + return Err(EINVAL); + } + let end = va.checked_add(size as u64).ok_or(EINVAL)?; + for v in (va..end).step_by(UAT_PGSZ) { + if self.table.translate(v)?.is_some() { + return Err(EEXIST); + } + } + // Retain the whole range first so partial mapping failures also unwind. + self.external.push(va..end, GFP_KERNEL)?; + self.table.map_pages(va..end, pa, prot, false) + } + + pub(crate) fn unmap_external(&mut self, range: core::ops::Range) -> Result { + self.table.unmap_pages(range.clone())?; + self.table.sync(); + self.table.invalidate(None); + crate::mem::sync(); + self.table.clear_invalidations(); + self.external + .retain(|r| r.start < range.start || r.end > range.end); + Ok(()) + } + + pub(crate) fn physical(&mut self, va: u64) -> Result { + self.table.translate(va)?.ok_or(EINVAL) + } + + pub(crate) fn alloc(&mut self, va: u64, size: usize, prot: crate::pgtable::Prot) -> Result { + if size == 0 || (va as usize | size) & (UAT_PGSZ - 1) != 0 { + return Err(EINVAL); + } + let end = va.checked_add(size as u64).ok_or(EINVAL)?; + // Failed mapping cleanup may retain an owned but partly unmapped + // region. Its address range must not be replaced or overlapped. + let previous = match self.regions.cursor_lower_bound(&va) { + Some(cursor) => { + if *cursor.current().0 < end { + return Err(EEXIST); + } + cursor.peek_prev().map(|(base, _)| *base) + } + None => self.regions.cursor_back().map(|cursor| *cursor.current().0), + }; + if previous.is_some_and(|base| { + let region = self.regions.get(&base).unwrap(); + base + (region.pages.len() * UAT_PGSZ) as u64 > va + }) { + return Err(EEXIST); + } + for v in (va..end).step_by(UAT_PGSZ) { + if self.table.translate(v)?.is_some() { + return Err(EEXIST); + } + } + let mut pages = KVec::new(); + for _ in 0..size / UAT_PGSZ { + pages.push(Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?, GFP_KERNEL)?; + } + let mut dirty_pages = KVec::new(); + dirty_pages.extend_with(pages.len(), Cell::new(true), GFP_KERNEL)?; + let node = RBTreeNode::new( + va, + FirmwareRegion { + va, + pages, + dirty_pages, + pending: Cell::new(true), + }, + GFP_KERNEL, + )?; + // Reserve dirty tracking before the allocation is owned or mapped. + self.dirty_regions.get_mut().push(va, GFP_KERNEL)?; + self.regions.insert(node); + let region = self.regions.get(&va).ok_or(EIO)?; + for (i, page) in region.pages.iter().enumerate() { + let v = va + (i * UAT_PGSZ) as u64; + if let Err(e) = self + .table + .map_pages(v..v + UAT_PGSZ as u64, page.phys(), prot, false) + { + if v != va { + self.table.unmap_pages(va..v)?; + } + self.table.sync(); + self.table.invalidate(None); + crate::mem::sync(); + self.table.clear_invalidations(); + self.dirty_regions.get_mut().pop(); + self.regions.remove(&va); + return Err(e); + } + } + Ok(()) + } + + pub(crate) fn write(&mut self, va: u64, bytes: &[u8]) -> Result { + let region = self.region(va, bytes.len())?; + if !bytes.is_empty() { + self.queue_region(region)?; + } + let mut offset = (va - region.va) as usize; + let mut remaining = bytes; + while !remaining.is_empty() { + let within = offset & (UAT_PGSZ - 1); + let size = remaining.len().min(UAT_PGSZ - within); + region.dirty_pages[offset / UAT_PGSZ].set(true); + region.pages[offset / UAT_PGSZ].with_pointer_into_page(within, size, |p| { + // SAFETY: These owned bytes are not yet published to firmware. + unsafe { + core::ptr::copy_nonoverlapping(remaining.as_ptr(), p, size); + } + Ok(()) + })?; + remaining = &remaining[size..]; + offset += size; + } + Ok(()) + } + + /// Initialize a whole owned page before its contents are published. + pub(crate) fn init_page( + &mut self, + va: u64, + initialize: impl FnOnce(&mut [u8]) -> Result, + ) -> Result { + if va & (UAT_PGSZ as u64 - 1) != 0 { + return Err(EINVAL); + } + let region = self.region(va, UAT_PGSZ)?; + self.queue_region(region)?; + region.dirty_pages[((va - region.va) as usize) / UAT_PGSZ].set(true); + region.pages[((va - region.va) as usize) / UAT_PGSZ].with_pointer_into_page( + 0, + UAT_PGSZ, + |ptr| { + // SAFETY: This page is owned by this space and initialization + // occurs before it is published to firmware. + initialize(unsafe { core::slice::from_raw_parts_mut(ptr, UAT_PGSZ) }) + }, + ) + } + + pub(crate) fn alloc_render_support(&mut self) -> Result { + use crate::g16_render::Addresses; + let a = Addresses::bootstrap(); + for (base, size) in a.private_regions() { + self.alloc(base, size, prot::PROT_FW_PRIV_RW)?; + for va in (base..base + size as u64).step_by(UAT_PGSZ) { + self.init_page(va, |bytes| a.private_page(va, bytes).ok_or(EINVAL))?; + } + } + for va in [a.tiling_shared_tail, a.fragment_shared_tail] { + self.alloc(va, UAT_PGSZ, prot::PROT_FW_SHARED_RW)?; + } + self.alloc( + crate::g16_render::SUPPORT_BASE, + 64 * UAT_PGSZ, + prot::PROT_FW_SHARED_RW, + )?; + for i in 0..64 { + let mut support = Addresses::bootstrap(); + support.support = crate::g16_render::SUPPORT_BASE + i * UAT_PGSZ as u64; + self.init_page(support.support, |bytes| { + support.private_page(support.support, bytes).ok_or(EINVAL) + })?; + } + Ok(()) + } + + /// Update host-owned bytes, preserving adjacent firmware-owned cache lines. + /// Cache maintenance completes once per range, not once per page. + pub(crate) fn write_live(&mut self, va: u64, bytes: &[u8]) -> Result { + self.update_live(va, bytes.len(), |ptr, offset, size| { + // SAFETY: update_live supplies an exclusively host-owned destination + // and a range within the source slice, with no alias between them. + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr().add(offset), ptr, size) }; + }) + } + + pub(crate) fn zero_live(&mut self, va: u64, size: usize) -> Result { + self.update_live(va, size, |ptr, _, size| { + // SAFETY: update_live supplies a valid host-owned byte range. + unsafe { core::ptr::write_bytes(ptr, 0, size) }; + }) + } + + fn update_live( + &mut self, + va: u64, + size: usize, + mut update: impl FnMut(*mut u8, usize, usize), + ) -> Result { + let region = self.region(va, size)?; + if size == 0 { + return Ok(()); + } + let start = (va - region.va) as usize; + let end = start + size; + let first = start / UAT_PGSZ; + let last = (end - 1) / UAT_PGSZ; + for index in first..=last { + // Unpublished initialization is still authoritative in CPU cache. + if region.dirty_pages[index].get() { + continue; + } + let base = index * UAT_PGSZ; + let within = start.saturating_sub(base); + let limit = (end - base).min(UAT_PGSZ); + region.pages[index].with_page_mapped(|p| { + for off in (within & !63..limit).step_by(64) { + // SAFETY: These cache lines lie within the mapped page. + unsafe { core::arch::asm!("dc ivac, {addr}", addr = in(reg) p.add(off)) }; + } + }); + } + crate::mem::sync(); + for index in first..=last { + let base = index * UAT_PGSZ; + let within = start.saturating_sub(base); + let limit = (end - base).min(UAT_PGSZ); + let dirty = region.dirty_pages[index].replace(false); + region.pages[index].with_page_mapped(|p| { + // SAFETY: The checked range lies in this mapped page. The + // caller owns these bytes; neighboring firmware bytes were + // invalidated before the batch's completion barrier above. + unsafe { update(p.add(within), base + within - start, limit - within) }; + // Publish all initialization on a dirty page, including zeros + // outside this write. Published pages need only touched lines. + let lines = if dirty { + 0..UAT_PGSZ + } else { + (within & !63)..limit + }; + for off in lines.step_by(64) { + // SAFETY: The cache line lies in this mapped page. + unsafe { core::arch::asm!("dc cvac, {addr}", addr = in(reg) p.add(off)) }; + } + }); + } + crate::mem::sync(); + Ok(()) + } + + /// Read one firmware-owned status word after invalidating the CPU view. + /// Host writes to this cache line must have been synced before publication. + pub(crate) fn read_u64(&self, va: u64) -> Result { + Ok(u64::from(self.read_u32(va)?) | (u64::from(self.read_u32(va + 4)?) << 32)) + } + + /// Called only after the context is drained and its hardware roots detached. + pub(crate) fn release_region(&mut self, va: u64) -> Result { + let size = self.regions.get(&va).ok_or(EINVAL)?.pages.len() * UAT_PGSZ; + self.table.unmap_pages(va..va + size as u64)?; + self.table.sync(); + self.table.invalidate(None); + crate::mem::sync(); + self.table.clear_invalidations(); + self.dirty_regions + .get_mut() + .retain(|pending| *pending != va); + self.regions.remove(&va); + Ok(()) + } + + pub(crate) fn read_u32(&self, va: u64) -> Result { + if va & 3 != 0 { + return Err(EINVAL); + } + let region = self.region(va, 4)?; + let offset = (va - region.va) as usize; + region.pages[offset / UAT_PGSZ].with_pointer_into_page(offset & (UAT_PGSZ - 1), 4, |p| { + // SAFETY: The aligned word lies in this owned mapped page. + // Firmware owns it and all CPU writes have reached PoC. + unsafe { + core::arch::asm!("dc ivac, {addr}", "dsb sy", addr = in(reg) p); + Ok(u32::from_le(p.cast::().read_volatile())) + } + }) + } + + pub(crate) fn sync(&self) { + for va in self.dirty_regions.borrow_mut().drain(..) { + let region = self.regions.get(&va).unwrap(); + region.pending.set(false); + for (page, dirty) in region.pages.iter().zip(®ion.dirty_pages) { + if dirty.replace(false) { + clean_page(page); + } + } + } + self.table.sync(); + self.table.invalidate(None); + crate::mem::sync(); + self.table.clear_invalidations(); + } + + /// Find the preceding allocation in logarithmic time, then validate the + /// entire access. Adjacent regions must not hide an out-of-bounds request. + fn region(&self, va: u64, size: usize) -> Result<&FirmwareRegion> { + let end = va.checked_add(size as u64).ok_or(EINVAL)?; + let base = match self.regions.cursor_lower_bound(&va) { + Some(cursor) if *cursor.current().0 == va => va, + Some(cursor) => *cursor.peek_prev().ok_or(EINVAL)?.0, + None => *self.regions.cursor_back().ok_or(EINVAL)?.current().0, + }; + let region = self.regions.get(&base).ok_or(EINVAL)?; + if end > base + (region.pages.len() * UAT_PGSZ) as u64 { + return Err(EINVAL); + } + Ok(region) + } + + fn queue_region(&self, region: &FirmwareRegion) -> Result { + if !region.pending.get() { + // Failure leaves both the bytes and queue membership unchanged. + self.dirty_regions + .borrow_mut() + .push(region.va, GFP_KERNEL)?; + region.pending.set(true); + } + Ok(()) + } +} + +impl Drop for FirmwareSpace { + fn drop(&mut self) { + // Only valid before publishing the graph or after firmware is stopped. + // Remove leaf mappings before their backing pages are released. + for region in self.regions.values() { + let _ = self + .table + .unmap_pages(region.va..region.va + (region.pages.len() * UAT_PGSZ) as u64); + } + for range in &self.external { + let _ = self.table.unmap_pages(range.clone()); + } + self.table.sync(); + self.table.invalidate(None); + crate::mem::sync(); + self.table.clear_invalidations(); + } +} + +fn clean_page(page: &Page) { + page.with_page_mapped(|p| { + for off in (0..UAT_PGSZ).step_by(64) { + // SAFETY: The cache line lies inside this owned page. + unsafe { + core::arch::asm!("dc cvac, {addr}", addr = in(reg) p.add(off)); + } + } + }); +} diff --git a/drivers/gpu/drm/asahi/m4_mem.rs b/drivers/gpu/drm/asahi/m4_mem.rs new file mode 100644 index 0000000000000000000000000000000000000000..d9c9d925f91092bf9730cfe96f23466e5c63aba3 --- /dev/null +++ b/drivers/gpu/drm/asahi/m4_mem.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors + +//! GPU memory publication and translation invalidation. +//! Cache-clean loops queue maintenance; publication boundaries complete it. + +#[inline(always)] +pub(crate) fn sync() { + // SAFETY: A system memory barrier changes no memory ownership. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; +} + +#[inline(always)] +pub(crate) fn tlbi_asid(asid: u8) { + // SAFETY: Invalidating a GPU ASID cannot expose stale translations. + unsafe { + core::arch::asm!(".arch armv8.4-a", "tlbi aside1os, {asid}", + asid = in(reg) u64::from(asid) << 48, options(nostack, preserves_flags)); + } +} + +/// Invalidate only these GPU virtual pages, including global firmware mappings. +/// VA operands use 4 KiB units even though the GPU page size is 16 KiB. +/// The caller publishes PTE writes before this and completes the batch after it. +pub(crate) fn tlbi_range(asid: Option, start: u64, end: u64) { + for va in (start..end).step_by(0x4000) { + let operand = (va >> 12) & 0x0000_0fff_ffff_fffc; + // SAFETY: These instructions only discard cached translations. + unsafe { + match asid { + Some(asid) => core::arch::asm!( + ".arch armv8.4-a", "tlbi vae1os, {operand}", + operand = in(reg) operand | (u64::from(asid) << 48), + options(nostack, preserves_flags)), + None => core::arch::asm!( + ".arch armv8.4-a", "tlbi vaae1os, {operand}", + operand = in(reg) operand, options(nostack, preserves_flags)), + } + } + } +} diff --git a/drivers/gpu/drm/asahi/m4_pgtable.rs b/drivers/gpu/drm/asahi/m4_pgtable.rs new file mode 100644 index 0000000000000000000000000000000000000000..eab02210d548e400e91a1e37b4e3ed5b58fe346c --- /dev/null +++ b/drivers/gpu/drm/asahi/m4_pgtable.rs @@ -0,0 +1,804 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from the Asahi UAT implementation and Niklas Sheth's G16 port. + +//! UAT Page Table management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the actual page tables by allocating raw memory pages from +//! the kernel page allocator. + +use core::cell::{Cell, RefCell}; +use core::fmt::Debug; +use core::mem::size_of; +use core::ops::Range; +use core::sync::atomic::{AtomicU64, Ordering}; + +use kernel::rbtree::RBTree; +use kernel::{addr::PhysicalAddr, error::Result, page::Page, prelude::*}; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{ + types::Owned, + uapi::{PF_R, PF_W, PF_X}, +}; + +use crate::util::align; + +/// Number of bits in a page offset. +pub(crate) const UAT_PGBIT: usize = 14; +/// UAT page size. +pub(crate) const UAT_PGSZ: usize = 1 << UAT_PGBIT; +/// UAT page offset mask. +pub(crate) const UAT_PGMSK: usize = UAT_PGSZ - 1; + +type Pte = AtomicU64; + +const PTE_BIT: usize = 3; // log2(sizeof(Pte)) +const PTE_SIZE: usize = 1 << PTE_BIT; + +/// Number of PTEs per page. +const UAT_NPTE: usize = UAT_PGSZ / size_of::(); + +/// Number of address bits to address a level +const UAT_LVBIT: usize = UAT_PGBIT - PTE_BIT; +/// Number of entries per level +const UAT_LVSZ: usize = UAT_NPTE; +/// Mask of level bits +const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; + +const UAT_LEVELS: usize = 3; + +const PTE_TYPE_BITS: u64 = 3; +const PTE_TYPE_LEAF_TABLE: u64 = 3; + +const UAT_NON_GLOBAL: u64 = 1 << 11; +const UAT_AP_SHIFT: u32 = 6; +#[cfg(CONFIG_DEV_COREDUMP)] +const UAT_AP_BITS: u64 = 3 << UAT_AP_SHIFT; +const UAT_HIGH_BITS_SHIFT: u32 = 52; +#[cfg(CONFIG_DEV_COREDUMP)] +const UAT_HIGH_BITS: u64 = 0xfff << UAT_HIGH_BITS_SHIFT; +const UAT_MEMATTR_SHIFT: u32 = 2; +#[cfg(CONFIG_DEV_COREDUMP)] +const UAT_MEMATTR_BITS: u64 = 7 << UAT_MEMATTR_SHIFT; + +const UAT_AF: u64 = 1 << 10; + +const MEMATTR_CACHED: u8 = 0; +const MEMATTR_DEV: u8 = 1; +const MEMATTR_UNCACHED: u8 = 2; + +const AP_FW_GPU: u8 = 0; +const AP_FW: u8 = 1; +const AP_GPU: u8 = 2; + +const HIGH_BITS_PXN: u16 = 1 << 1; +const HIGH_BITS_UXN: u16 = 1 << 2; +const HIGH_BITS_GPU_ACCESS: u16 = 1 << 3; + +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) const PTE_ADDR_BITS: u64 = (!UAT_PGMSK as u64) & (!UAT_HIGH_BITS); + +#[derive(Debug, Copy, Clone)] +pub(crate) struct Prot { + memattr: u8, + ap: u8, + high_bits: u16, +} + +// Firmware + GPU access +const PROT_FW_GPU_NA: Prot = Prot::from_bits(AP_FW_GPU, 0, 0); +const _PROT_FW_GPU_RO: Prot = Prot::from_bits(AP_FW_GPU, 0, 1); +const _PROT_FW_GPU_WO: Prot = Prot::from_bits(AP_FW_GPU, 1, 0); + +// Firmware only access +const PROT_FW_RO: Prot = Prot::from_bits(AP_FW, 0, 0); +const _PROT_FW_NA: Prot = Prot::from_bits(AP_FW, 0, 1); +const PROT_FW_RW: Prot = Prot::from_bits(AP_FW, 1, 0); + +// GPU only access +const PROT_GPU_RO: Prot = Prot::from_bits(AP_GPU, 0, 0); +const PROT_GPU_WO: Prot = Prot::from_bits(AP_GPU, 0, 1); +const PROT_GPU_RW: Prot = Prot::from_bits(AP_GPU, 1, 0); +const _PROT_GPU_NA: Prot = Prot::from_bits(AP_GPU, 1, 1); + +#[cfg(CONFIG_DEV_COREDUMP)] +const PF_RW: u32 = PF_R | PF_W; +#[cfg(CONFIG_DEV_COREDUMP)] +const PF_RX: u32 = PF_R | PF_X; + +// For crash dumps +#[cfg(CONFIG_DEV_COREDUMP)] +const PROT_TO_PERMS_FW: [[u32; 4]; 4] = [ + [0, 0, 0, PF_RW], + [0, PF_RW, 0, PF_RW], + [PF_RX, PF_RX, 0, PF_R], + [PF_RX, PF_RW, 0, PF_R], +]; +#[cfg(CONFIG_DEV_COREDUMP)] +const PROT_TO_PERMS_OS: [[u32; 4]; 4] = [ + [0, PF_R, PF_W, PF_RW], + [PF_R, 0, PF_RW, PF_RW], + [0, 0, 0, 0], + [0, 0, 0, 0], +]; + +pub(crate) mod prot { + pub(crate) use super::Prot; + use super::*; + + /// Firmware MMIO R/W + pub(crate) const PROT_FW_MMIO_RW: Prot = PROT_FW_RW.memattr(MEMATTR_DEV); + /// Firmware MMIO R/O + pub(crate) const PROT_FW_MMIO_RO: Prot = PROT_FW_RO.memattr(MEMATTR_DEV); + /// Firmware shared (uncached) RW + pub(crate) const PROT_FW_SHARED_RW: Prot = PROT_FW_RW.memattr(MEMATTR_UNCACHED); + /// Firmware shared (uncached) RO + pub(crate) const PROT_FW_SHARED_RO: Prot = PROT_FW_RO.memattr(MEMATTR_UNCACHED); + /// Firmware private (cached) RW + pub(crate) const PROT_FW_PRIV_RW: Prot = PROT_FW_RW.memattr(MEMATTR_CACHED); + /// GPU shared/coherent RW + pub(crate) const PROT_GPU_SHARED_RW: Prot = PROT_GPU_RW.memattr(MEMATTR_UNCACHED); + /// GPU shared/coherent RO + pub(crate) const PROT_GPU_SHARED_RO: Prot = PROT_GPU_RO.memattr(MEMATTR_UNCACHED); + /// GPU shared/coherent WO + pub(crate) const PROT_GPU_SHARED_WO: Prot = PROT_GPU_WO.memattr(MEMATTR_UNCACHED); +} + +impl Prot { + const fn from_bits(ap: u8, uxn: u16, pxn: u16) -> Self { + assert!(uxn <= 1); + assert!(pxn <= 1); + assert!(ap <= 3); + + Prot { + high_bits: HIGH_BITS_GPU_ACCESS | (pxn * HIGH_BITS_PXN) | (uxn * HIGH_BITS_UXN), + memattr: 0, + ap, + } + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) const fn from_pte(pte: u64) -> Self { + Prot { + high_bits: (pte >> UAT_HIGH_BITS_SHIFT) as u16, + ap: ((pte & UAT_AP_BITS) >> UAT_AP_SHIFT) as u8, + memattr: ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8, + } + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) const fn elf_flags(&self) -> u32 { + let ap = (self.ap & 3) as usize; + let uxn = if self.high_bits & HIGH_BITS_UXN != 0 { + 1 + } else { + 0 + }; + let pxn = if self.high_bits & HIGH_BITS_PXN != 0 { + 1 + } else { + 0 + }; + let gpu = self.high_bits & HIGH_BITS_GPU_ACCESS != 0; + + // Format: + // [12 top bits of PTE] [12 bottom bits of PTE] [5 bits pad] [ELF RWX] + let mut perms = if gpu { + PROT_TO_PERMS_OS[ap][(uxn << 1) | pxn] + } else { + PROT_TO_PERMS_FW[ap][(uxn << 1) | pxn] + }; + + perms |= ((self.as_pte() >> 52) << 20) as u32; + perms |= ((self.as_pte() & 0xfff) << 8) as u32; + + perms + } + + const fn memattr(&self, memattr: u8) -> Self { + Self { memattr, ..*self } + } + + const fn as_pte(&self) -> u64 { + (self.ap as u64) << UAT_AP_SHIFT + | (self.high_bits as u64) << UAT_HIGH_BITS_SHIFT + | (self.memattr as u64) << UAT_MEMATTR_SHIFT + | UAT_AF + } +} + +impl Default for Prot { + fn default() -> Self { + PROT_FW_GPU_NA + } +} + +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) struct DumpedPage { + pub(crate) iova: u64, + pub(crate) pte: u64, + pub(crate) data: Option>, +} + +enum Walk { + Read, + Write, + Allocate, +} + +struct OwnedTable { + phys: PhysicalAddr, + parent: PhysicalAddr, + index: usize, +} + +pub(crate) struct UatPageTable { + ttb: PhysicalAddr, + ttb_owned: bool, + va_range: Range, + oas_mask: u64, + ias_mask: u64, + owned_tables: KVec, + // Membership survives publication; the queue contains only pending cleans. + dirty_tables: RBTree>, + pending_tables: RefCell>, + invalidations: RefCell>>, +} + +impl UatPageTable { + /// Allocate a root with the address width of this GPU generation. + /// G16 has 64 top-level entries (42 address bits per TTBR), while + /// G13/G14 use 8 entries (39 bits per TTBR). + pub(crate) fn new_with_ias(oas: u32, ias: usize) -> Result { + if !(UAT_PGBIT..=UAT_PGBIT + UAT_LEVELS * UAT_LVBIT).contains(&ias) + || !(UAT_PGBIT as u32..=52).contains(&oas) + { + return Err(EINVAL); + } + pr_debug!("UATPageTable::new: oas={} ias={}\n", oas, ias); + let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let mut dirty_tables = RBTree::new(); + dirty_tables.try_create_and_insert(ttb_page.phys(), Cell::new(true), GFP_KERNEL)?; + let mut pending_tables = KVec::new(); + pending_tables.push(ttb_page.phys(), GFP_KERNEL)?; + let ttb = Page::into_phys(ttb_page); + Ok(UatPageTable { + ttb, + ttb_owned: true, + va_range: 0..(1u64 << ias), + oas_mask: (1u64 << oas) - 1, + ias_mask: (1u64 << ias) - 1, + owned_tables: KVec::new(), + dirty_tables, + pending_tables: RefCell::new(pending_tables), + invalidations: RefCell::new(KVec::new()), + }) + } + + /// Borrow a firmware root without owning its existing child tables. + pub(crate) fn new_with_ttb_and_ias( + ttb: PhysicalAddr, + va_range: Range, + oas: u32, + ias: usize, + ) -> Result { + if !(UAT_PGBIT..=UAT_PGBIT + UAT_LEVELS * UAT_LVBIT).contains(&ias) + || !(UAT_PGBIT as u32..=52).contains(&oas) + || va_range.is_empty() + || va_range.end - va_range.start > 1u64 << ias + { + return Err(EINVAL); + } + pr_debug!( + "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} oas={}\n", + ttb, + va_range, + oas + ); + if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { + return Err(EINVAL); + } + if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + // SAFETY: The TTB is should remain valid (if properly mapped), as it is bootloader-managed. + if unsafe { Page::borrow_phys(&ttb) }.is_none() { + pr_err!( + "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", + ttb + ); + return Err(EIO); + } + + Ok(UatPageTable { + ttb, + ttb_owned: false, + va_range, + oas_mask: (1u64 << oas) - 1, + ias_mask: (1u64 << ias) - 1, + owned_tables: KVec::new(), + dirty_tables: RBTree::new(), + pending_tables: RefCell::new(KVec::new()), + invalidations: RefCell::new(KVec::new()), + }) + } + + /// Queue cache cleaning for edited tables, including borrowed ancestors. + /// The enclosing publication batch supplies the completion barrier. + pub(crate) fn clean(&self) { + for phys in self.pending_tables.borrow_mut().drain(..) { + self.dirty_tables.get(&phys).unwrap().set(false); + // SAFETY: Every recorded table is live until this owner is dropped. + let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + page.with_page_mapped(|p| { + for off in (0..UAT_PGSZ).step_by(64) { + // SAFETY: This cache line lies within the mapped table page. + unsafe { + core::arch::asm!("dc cvac, {addr}", addr = in(reg) p.add(off)); + } + } + }); + } + } + + pub(crate) fn sync(&self) { + self.clean(); + crate::mem::sync(); + } + + /// Record before editing so an allocation failure cannot hide a changed + /// mapping. Adjacent edits share one invalidation interval. + fn changed(&mut self, range: Range) -> Result { + if range.is_empty() + || range.start < self.va_range.start + || range.end > self.va_range.end + || (range.start | range.end) & UAT_PGMSK as u64 != 0 + { + return Err(EINVAL); + } + let ranges = self.invalidations.get_mut(); + if let Some(last) = ranges.last_mut() { + if last.start <= range.end && range.start <= last.end { + last.start = last.start.min(range.start); + last.end = last.end.max(range.end); + return Ok(()); + } + } + ranges.push(range, GFP_KERNEL)?; + Ok(()) + } + + /// PTE cleaning must have completed before invalidation. None targets + /// global firmware mappings by VA across all ASIDs, never the whole TLB. + pub(crate) fn invalidate(&self, asid: Option) { + for range in self.invalidations.borrow().iter() { + crate::mem::tlbi_range(asid, range.start, range.end); + } + } + + pub(crate) fn reserve_invalidation(&mut self) -> Result { + self.invalidations.get_mut().reserve(1, GFP_KERNEL)?; + Ok(()) + } + + pub(crate) fn clear_invalidations(&self) { + self.invalidations.borrow_mut().clear(); + } + + // Record dirtiness before writing a PTE so allocation failure cannot leave + // an untracked edit. Read-only walks neither allocate nor dirty metadata. + fn mark_dirty(&mut self, phys: PhysicalAddr) -> Result { + match self.dirty_tables.get(&phys) { + Some(dirty) if dirty.get() => return Ok(()), + Some(_) => {} + None => { + self.dirty_tables + .try_create_and_insert(phys, Cell::new(false), GFP_KERNEL)?; + } + } + // Queue before setting membership or editing a PTE. A failed reserve + // leaves the table clean and permits a later attempt to enqueue it. + self.pending_tables.get_mut().push(phys, GFP_KERNEL)?; + self.dirty_tables.get(&phys).unwrap().set(true); + Ok(()) + } + + pub(crate) fn ttb(&self) -> PhysicalAddr { + self.ttb + } + + fn with_pages(&mut self, iova_range: Range, walk: Walk, mut cb: F) -> Result + where + F: FnMut(u64, &[Pte]) -> Result, + { + let alloc = matches!(walk, Walk::Allocate); + let write = !matches!(walk, Walk::Read); + pr_debug!( + "UATPageTable::with_pages: {:#x?} alloc={}\n", + iova_range, + alloc + ); + if (iova_range.start | iova_range.end) & (UAT_PGMSK as u64) != 0 { + pr_err!( + "UATPageTable::with_pages: iova range not aligned: {:#x?}\n", + iova_range + ); + return Err(EINVAL); + } + + if iova_range.is_empty() { + return Ok(()); + } + + let mut iova = iova_range.start & self.ias_mask; + let mut last_iova = iova; + // Handle the case where iova_range.end is just at the top boundary of the IAS + let end = ((iova_range.end - 1) & self.ias_mask) + 1; + + let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + pt_addr[UAT_LEVELS - 1] = Some(self.ttb); + + 'outer: while iova < end { + pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); + let addr_diff = last_iova ^ iova; + for level in (0..UAT_LEVELS - 1).rev() { + // If the iova has changed at this level or above, invalidate the physaddr + if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { + if pt_addr[level].take().is_some() { + pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); + } + } + } + last_iova = iova; + for level in (0..UAT_LEVELS - 1).rev() { + // Fetch the page table base address for this level + if pt_addr[level].is_none() { + let phys = pt_addr[level + 1].unwrap(); + pr_debug!( + "UATPageTable::with_pages: need level {}, parent phys {:#x}\n", + level, + phys + ); + let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) + as usize; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; + pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); + pt_addr[level] = + upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_pointer_into_page() ensures the pointer is valid, + // and our index is aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + self.mark_dirty(phys)?; + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + let pt_paddr = pt_page.phys(); + self.owned_tables.push(OwnedTable { + phys: pt_paddr, + parent: phys, + index: upidx, + }, GFP_KERNEL)?; + let _ = Page::into_phys(pt_page); + // The new table's zero entries also need publication. + // Ownership has transferred before this fallible step. + self.mark_dirty(pt_paddr)?; + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + upte.store(upte_val, Ordering::Relaxed); + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || !alloc { + pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + })?; + pr_debug!( + "UATPageTable::with_pages: level {} PT {:#x?}\n", + level, + pt_addr[level] + ); + } + // If we don't have a page table, skip this entire level + if pt_addr[level].is_none() { + let block = 1 << (UAT_PGBIT + UAT_LVBIT * (level + 1)); + let old = iova; + iova = align(iova + 1, block); + pr_debug!( + "UATPageTable::with_pages: skip {:#x} {:#x} -> {:#x}\n", + block, + old, + iova + ); + continue 'outer; + } + } + + let idx = ((iova >> UAT_PGBIT as u64) & UAT_LVMSK) as usize; + let max_count = UAT_NPTE - idx; + let count = (((end - iova) >> UAT_PGBIT) as usize).min(max_count); + let phys = pt_addr[0].unwrap(); + pr_debug!( + "UATPageTable::with_pages: leaf PT at {:#x} idx {:#x} count {:#x} iova {:#x}\n", + phys, + idx, + count, + iova + ); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + if write { + self.mark_dirty(phys)?; + } + let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; + pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let ptep = p as *const _ as *const Pte; + // SAFETY: We know this is a valid pointer to PTEs and the range is valid and + // checked by with_pointer_into_page(). + let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; + cb(iova, ptes)?; + Ok(()) + })?; + + let block = 1 << (UAT_PGBIT + UAT_LVBIT); + iova = align(iova + 1, block); + } + + Ok(()) + } + + fn pte_bits(&self) -> u64 { + if self.ttb_owned { + // Owned page tables are userspace, so non-global + PTE_TYPE_LEAF_TABLE | UAT_NON_GLOBAL + } else { + // The sole non-owned page table is kernelspace, so global + PTE_TYPE_LEAF_TABLE + } + } + + pub(crate) fn map_pages( + &mut self, + iova_range: Range, + mut phys: PhysicalAddr, + prot: Prot, + one_page: bool, + ) -> Result { + pr_debug!( + "UATPageTable::map_pages: {:#x?} {:#x?} {:?}\n", + iova_range, + phys, + prot + ); + if phys & (UAT_PGMSK as PhysicalAddr) != 0 { + pr_err!("UATPageTable::map_pages: phys not aligned: {:#x?}\n", phys); + return Err(EINVAL); + } + + self.changed(iova_range.clone())?; + let pte_bits = self.pte_bits(); + + self.with_pages(iova_range, Walk::Allocate, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev != 0 { + pr_err!( + "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + } + pte.store(phys | prot.as_pte() | pte_bits, Ordering::Relaxed); + if !one_page { + phys += UAT_PGSZ as PhysicalAddr; + } + } + Ok(()) + }) + } + + /// Resolve a mapped byte address without allocating page tables. + pub(crate) fn translate(&mut self, iova: u64) -> Result> { + let page = iova & !(UAT_PGMSK as u64); + let end = page.checked_add(UAT_PGSZ as u64).ok_or(EINVAL)?; + if page < self.va_range.start || end > self.va_range.end { + return Err(EINVAL); + } + let mut result = None; + let mask = self.oas_mask & !(UAT_PGMSK as u64); + self.with_pages(page..end, Walk::Read, |_, ptes| { + let pte = ptes[0].load(Ordering::Relaxed); + if pte & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + result = Some((pte & mask) | (iova & UAT_PGMSK as u64)); + } + Ok(()) + })?; + Ok(result) + } + + /// Save one existing leaf while a Work temporarily substitutes owned RAM. + #[cfg(test)] + pub(crate) fn leaf(&mut self, address: u64) -> Result { + if address & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let mut value = 0; + self.with_pages(address..address + UAT_PGSZ as u64, Walk::Read, |_, ptes| { + value = ptes[0].load(Ordering::Relaxed); + Ok(()) + })?; + if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + return Err(EINVAL); + } + Ok(value) + } + + /// Restore a leaf obtained from this VM by `leaf`, before its backing dies. + #[cfg(test)] + pub(crate) fn restore_leaf(&mut self, address: u64, value: u64) -> Result { + if address & UAT_PGMSK as u64 != 0 || value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + return Err(EINVAL); + } + self.with_pages( + address..address + UAT_PGSZ as u64, + Walk::Write, + |_, ptes| { + ptes[0].store(value, Ordering::Relaxed); + Ok(()) + }, + ) + } + + pub(crate) fn unmap_pages(&mut self, iova_range: Range) -> Result { + self.changed(iova_range.clone())?; + pr_debug!("UATPageTable::unmap_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, Walk::Write, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) & PTE_TYPE_LEAF_TABLE == 0 { + pr_err!( + "UATPageTable::unmap_pages: Page at IOVA {:#x} already unmapped\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + pte.store(0, Ordering::Relaxed); + } + Ok(()) + }) + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn dump_pages(&mut self, iova_range: Range) -> Result> { + let mut pages = KVVec::new(); + let oas_mask = self.oas_mask; + let iova_base = self.va_range.start & !self.ias_mask; + self.with_pages(iova_range, Walk::Read, |iova, ptes| { + let iova = iova | iova_base; + for (idx, ppte) in ptes.iter().enumerate() { + let pte = ppte.load(Ordering::Relaxed); + if (pte & PTE_TYPE_LEAF_TABLE) != PTE_TYPE_LEAF_TABLE { + continue; + } + let memattr = ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8; + + if !(memattr == MEMATTR_CACHED || memattr == MEMATTR_UNCACHED) { + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: None, + }, + GFP_KERNEL, + )?; + continue; + } + let phys = pte & oas_mask & (!UAT_PGMSK as u64); + // SAFETY: GPU pages are either firmware/preallocated pages + // (which the kernel isn't concerned with and are either in + // the page map or not, and if they aren't, borrow_phys() + // will fail), or GPU page table pages (which we own), + // or GEM buffer pages (which are locked while they are + // mapped in the page table), so they should be safe to + // borrow. + // + // This does trust the firmware not to have any weird + // mappings in its own internal page tables, but since + // those are managed by the uPPL which is privileged anyway, + // this trust does not actually extend any trust boundary. + let src_page = match unsafe { Page::borrow_phys(&phys) } { + Some(page) => page, + None => { + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: None, + }, + GFP_KERNEL, + )?; + continue; + } + }; + let dst_page = Page::alloc_page(GFP_KERNEL)?; + src_page.with_page_mapped(|psrc| -> Result { + // SAFETY: This could technically still have a data race with the firmware + // or other driver code (or even userspace with timestamp buffers), but while + // the Rust language technically says this is UB, in the real world, using + // atomic reads for this is guaranteed to never cause any harmful effects + // other than possibly reading torn/unreliable data. At least on ARM64 anyway. + // + // (Yes, I checked with Rust people about this. ~~ Lina) + // + let src_items = unsafe { + core::slice::from_raw_parts( + psrc as *const AtomicU64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + dst_page.with_page_mapped(|pdst| -> Result { + // SAFETY: We own the destination page, so it is safe to view its contents + // as a u64 slice. + let dst_items = unsafe { + core::slice::from_raw_parts_mut( + pdst as *mut u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + for (si, di) in src_items.iter().zip(dst_items.iter_mut()) { + *di = si.load(Ordering::Relaxed); + } + Ok(()) + })?; + Ok(()) + })?; + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: Some(dst_page), + }, + GFP_KERNEL, + )?; + } + Ok(()) + })?; + Ok(pages) + } +} + +impl Drop for UatPageTable { + fn drop(&mut self) { + pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); + // Free only the child tables allocated by this owner. G16 extends + // firmware-owned child tables inside its host VA range; walking and + // freeing the whole range would release reserved firmware memory. + while let Some(table) = self.owned_tables.pop() { + // SAFETY: Parents precede children in the allocation list. A + // parent is either still owned here or is bootloader-reserved. + let parent = unsafe { Page::borrow_phys_unchecked(&table.parent) }; + let _ = parent.with_pointer_into_page(table.index * PTE_SIZE, PTE_SIZE, |p| { + // SAFETY: This checked slot is an aligned atomic page-table entry. + let pte = unsafe { &*p.cast::() }; + pte.store(0, Ordering::Relaxed); + Ok(()) + }); + // SAFETY: Each tracked table was transferred once with into_phys. + unsafe { Page::from_phys(table.phys) }; + } + if self.ttb_owned { + pr_debug!("UATPageTable::drop: Free TTB {:#x}\n", self.ttb); + // SAFETY: If we own the ttb, it was allocated with Page::into_phys(). + unsafe { + Page::from_phys(self.ttb); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/m4_util.rs b/drivers/gpu/drm/asahi/m4_util.rs new file mode 100644 index 0000000000000000000000000000000000000000..708a3f5f124ec074beb053eb32ac185f12aedc97 --- /dev/null +++ b/drivers/gpu/drm/asahi/m4_util.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright The Gravity Linux Contributors +// Adapted from the Asahi utility functions. + +//! Miscellaneous utility functions + +use core::ops::{Add, BitAnd, Div, Not, Sub}; +use kernel::prelude::*; + +/// Aligns an integer type to a power of two. +pub(crate) fn align(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Add + + Sub + + Div + + core::cmp::PartialEq, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + assert!((b & (b - one)) == def); + + (a + b - one) & !(b - one) +} + +pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} + +pub(crate) struct Reader<'a> { + buffer: &'a [u8], + offset: usize, +} + +impl<'a> Reader<'a> { + pub(crate) fn new(buffer: &'a [u8]) -> Self { + Reader { buffer, offset: 0 } + } + + pub(crate) fn read_up_to(&mut self, max_size: usize) -> Result { + let end = self.offset.checked_add(max_size).ok_or(EINVAL)?; + let source = self.buffer.get(self.offset..end).ok_or(EINVAL)?; + let size = core::mem::size_of::().min(max_size); + // Forward-compatible records may extend the known structure only + // with zero bytes; shorter records retain zero defaults. + if source[size..].iter().any(|byte| *byte != 0) { + return Err(EINVAL); + } + let mut obj: T = Default::default(); + // SAFETY: T admits every bit pattern and the slice fits the object. + let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + dst.copy_from_slice(&source[..size]); + self.offset = end; + Ok(obj) + } + + pub(crate) fn read(&mut self) -> Result { + self.read_up_to(core::mem::size_of::()) + } + + pub(crate) fn is_empty(&self) -> bool { + self.offset >= self.buffer.len() + } +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 7b8f809cdc412d390d3e7657db7e7b00185b7455..b37f59ca75f8fd8afda2c312c9b04d541176712f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -20,5 +20,5 @@ #define BRCMF_BSS_INFO_MIN_VERSION 109 /* min ver of brcmf_bss_info_le struct */ -#define BRCMF_BSS_INFO_MAX_VERSION 112 /* max ver of brcmf_bss_info_le struct */ +#define BRCMF_BSS_INFO_MAX_VERSION 116 /* max ver of brcmf_bss_info_le struct */ #define BRCMF_BSS_RSSI_ON_CHANNEL 0x0004 diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index a3343c49c6af60839242e5f1468cd78276b7b615..92ef5316e7fbe1e9868cd321c6a3b9ad4cc2bbf5 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -43,5 +43,9 @@ #define APPLE_ANS_IOCQ_DB 0x100c -#define APPLE_ANS_MAX_PEND_CMDS_CTRL 0x1210 +#define APPLE_ANS_T8132_IOQ_CMDS 0x1200 +#define APPLE_ANS_T8132_IOQ_CQES 0x1208 +#define APPLE_ANS_T8132_IOQ_SIZE 0x1210 + +#define APPLE_ANS_T8103_MAX_PEND_CMDS_CTRL 0x1210 #define APPLE_ANS_BOOT_STATUS 0x1300 @@ -168,4 +172,5 @@ struct apple_nvme_iod { struct apple_nvme_hw { bool has_lsq_nvmmu; + bool needs_ioq_register; u32 max_queue_depth; }; @@ -1155,7 +1160,14 @@ static void apple_nvme_reset_work(struct work_struct *work) /* Allow as many pending command as possible for both queues */ - writel(anv->hw->max_queue_depth - | (anv->hw->max_queue_depth << 16), anv->mmio_nvme - + APPLE_ANS_MAX_PEND_CMDS_CTRL); + if (!anv->hw->needs_ioq_register) { + /* + * Since T8132, this is replaced by IOQ_SIZE which is + * located at the same offset and is handled with the + * IO queue setup further below. + */ + writel(anv->hw->max_queue_depth + | (anv->hw->max_queue_depth << 16), anv->mmio_nvme + + APPLE_ANS_T8103_MAX_PEND_CMDS_CTRL); + } /* Setup the NVMMU for the maximum admin and IO queue depth */ @@ -1212,4 +1224,14 @@ static void apple_nvme_reset_work(struct work_struct *work) goto out_remove_cq; + if (anv->hw->needs_ioq_register) { + writeq(anv->ioq.cq_dma_addr, + anv->mmio_nvme + APPLE_ANS_T8132_IOQ_CQES); + writeq(anv->ioq.sq_dma_addr, + anv->mmio_nvme + APPLE_ANS_T8132_IOQ_CMDS); + writel((anv->hw->max_queue_depth - 1) + | ((anv->hw->max_queue_depth - 1) << 16), anv->mmio_nvme + + APPLE_ANS_T8132_IOQ_SIZE); + } + apple_nvme_init_queue(&anv->ioq); nr_io_queues = 1; @@ -1754,4 +1776,5 @@ static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend, static const struct apple_nvme_hw apple_nvme_t8015_hw = { .has_lsq_nvmmu = false, + .needs_ioq_register = false, .max_queue_depth = 16, }; @@ -1759,4 +1782,11 @@ static const struct apple_nvme_hw apple_nvme_t8015_hw = { static const struct apple_nvme_hw apple_nvme_t8103_hw = { .has_lsq_nvmmu = true, + .needs_ioq_register = false, + .max_queue_depth = 64, +}; + +static const struct apple_nvme_hw apple_nvme_t8132_hw = { + .has_lsq_nvmmu = true, + .needs_ioq_register = true, .max_queue_depth = 64, }; @@ -1765,4 +1795,5 @@ static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, + { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, {}, diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 420db1255346f987b516634769a95211e2f94729..42af0a08be6aa1be35239102b646ea088e4cc824 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -48,4 +48,5 @@ config PCIE_APPLE depends on PAGE_SIZE_16KB || COMPILE_TEST select PCI_HOST_COMMON + select PCI_PWRCTRL_GENERIC select IRQ_MSI_LIB help diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index d962724750840de0288f856a9d5e65db2856d5c4..ffe81b1ff4bd10b48bee191af83d33b4e5a91fcc 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -31,4 +31,5 @@ #include #include +#include #include "pci-host-common.h" @@ -904,4 +905,26 @@ static int apple_pcie_init(struct pci_config_window *cfg) return -ENOENT; + // Ensure all ports can be powered on before attempting initialization + for_each_available_child_of_node_scoped(dev->of_node, of_port) { + ret = pci_pwrctrl_create_device(of_port, pcie->dev); + if (ret) { + dev_err(pcie->dev, "Failed to create pwrctrl device: %pe\n", ERR_PTR(ret)); + return ret; + } + + ret = pci_pwrctrl_power_on_device(of_port); + if (ret) { + if (ret != -EPROBE_DEFER) { + dev_err(pcie->dev, + "Failed to power on device: %pe\n", + ERR_PTR(ret)); + for_each_available_child_of_node_scoped(dev->of_node, of_port) { + pci_pwrctrl_destroy_device(of_port); + } + } + return ret; + } + } + for_each_available_child_of_node_scoped(dev->of_node, of_port) { ret = apple_pcie_setup_port(pcie, of_port); diff --git a/drivers/pci/pwrctrl/core.c b/drivers/pci/pwrctrl/core.c index cd08d590483b8653d9a8c3ac6e1ae62eb2b9ff02..050e0b089e76b74849277a52224b3520dabc45b2 100644 --- a/drivers/pci/pwrctrl/core.c +++ b/drivers/pci/pwrctrl/core.c @@ -150,5 +150,5 @@ static int __pci_pwrctrl_power_off_device(struct device *dev) } -static void pci_pwrctrl_power_off_device(struct device_node *np) +void pci_pwrctrl_power_off_device(struct device_node *np) { struct platform_device *pdev; @@ -172,4 +172,5 @@ static void pci_pwrctrl_power_off_device(struct device_node *np) platform_device_put(pdev); } +EXPORT_SYMBOL_GPL(pci_pwrctrl_power_off_device); /** @@ -205,5 +206,5 @@ static int __pci_pwrctrl_power_on_device(struct device *dev) * make sure its driver is bound. */ -static int pci_pwrctrl_power_on_device(struct device_node *np) +int pci_pwrctrl_power_on_device(struct device_node *np) { struct platform_device *pdev; @@ -234,4 +235,5 @@ static int pci_pwrctrl_power_on_device(struct device_node *np) return ret; } +EXPORT_SYMBOL_GPL(pci_pwrctrl_power_on_device); /** @@ -315,6 +317,5 @@ static bool pci_pwrctrl_is_required(struct device_node *np) } -static int pci_pwrctrl_create_device(struct device_node *np, - struct device *parent) +int pci_pwrctrl_create_device(struct device_node *np, struct device *parent) { struct platform_device *pdev; @@ -348,4 +349,5 @@ static int pci_pwrctrl_create_device(struct device_node *np, return 0; } +EXPORT_SYMBOL_GPL(pci_pwrctrl_create_device); /** @@ -376,5 +378,5 @@ int pci_pwrctrl_create_devices(struct device *parent) EXPORT_SYMBOL_GPL(pci_pwrctrl_create_devices); -static void pci_pwrctrl_destroy_device(struct device_node *np) +void pci_pwrctrl_destroy_device(struct device_node *np) { struct platform_device *pdev; @@ -392,4 +394,5 @@ static void pci_pwrctrl_destroy_device(struct device_node *np) of_node_clear_flag(np, OF_POPULATED); } +EXPORT_SYMBOL_GPL(pci_pwrctrl_destroy_device); /** diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 9eb0a1d106ff93dbfd153a00331205299ce020f1..9c223ceb906486cd8184a550ac1d654b1556bf19 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -616,4 +616,6 @@ struct atcphy_hw { * @dp_link_rate: DisplayPort link rate * @pipehandler_up: True if the PIPE mux ("pipehandler") is set to USB3 or USB4 mode + * @fixed_usb3: True for a permanently attached, host-only USB3 connection + * @fixed_lane_swap: True if a fixed USB3 connection uses the swapped orientation * @regs: Memory-mapped registers * @regs.core: Core registers @@ -652,4 +654,6 @@ struct apple_atcphy { bool swap_lanes; bool pipehandler_up; + bool fixed_usb3; + bool fixed_lane_swap; struct { @@ -1300,5 +1304,5 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode FIELD_PREP(ACIOPHY_CROSSBAR_PROTOCOL, mode_cfg->crossbar)); - if (mode_cfg->set_swap) + if (mode_cfg->set_swap || atcphy->fixed_lane_swap) core_set32(atcphy, ATCPHY_MISC, ATCPHY_MISC_LANE_SWAP); else @@ -2009,4 +2013,5 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode { struct apple_atcphy *atcphy = phy_get_drvdata(phy); + int ret; guard(mutex)(&atcphy->lock); @@ -2020,4 +2025,10 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode return 0; + if (atcphy->fixed_usb3 && atcphy->mode == APPLE_ATCPHY_MODE_OFF) { + ret = atcphy_configure(atcphy, APPLE_ATCPHY_MODE_USB3); + if (ret) + return ret; + } + switch (mode) { case PHY_MODE_USB_HOST: @@ -2190,4 +2201,9 @@ static int atcphy_dwc3_reset_deassert(struct reset_controller_dev *rcdev, unsign guard(mutex)(&atcphy->lock); + if (atcphy->fixed_usb3) { + atcphy_usb2_power_on(atcphy); + set32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + } + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); @@ -2437,10 +2453,12 @@ static int atcphy_probe_finalize(struct apple_atcphy *atcphy) if (ret) return dev_err_probe(atcphy->dev, ret, "Probing rcdev failed"); - ret = atcphy_probe_mux(atcphy); - if (ret) - return dev_err_probe(atcphy->dev, ret, "Probing mux failed"); - ret = atcphy_probe_switch(atcphy); - if (ret) - return dev_err_probe(atcphy->dev, ret, "Probing switch failed"); + if (!atcphy->fixed_usb3) { + ret = atcphy_probe_mux(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing mux failed"); + ret = atcphy_probe_switch(atcphy); + if (ret) + return dev_err_probe(atcphy->dev, ret, "Probing switch failed"); + } ret = atcphy_probe_phy(atcphy); if (ret) @@ -2478,4 +2496,6 @@ static int atcphy_probe(struct platform_device *pdev) atcphy->mode = APPLE_ATCPHY_MODE_OFF; atcphy->pipehandler_up = false; + atcphy->fixed_usb3 = device_property_read_bool(dev, "apple,fixed-usb3"); + atcphy->fixed_lane_swap = device_property_read_bool(dev, "apple,fixed-lane-swap"); return atcphy_probe_finalize(atcphy); diff --git a/drivers/phy/apple/dptx.c b/drivers/phy/apple/dptx.c index 5e58f149ce96afef7ad3b9c444c5cb81aff9cfc1..dbe28c4131d68a0e5bb57665a70ade015ba1dc9f 100644 --- a/drivers/phy/apple/dptx.c +++ b/drivers/phy/apple/dptx.c @@ -17,9 +17,12 @@ #include "linux/of.h" #include +#include #include #include +#include #include #include #include +#include #include @@ -32,4 +35,5 @@ enum apple_dptx_type { DPTX_PHY_T8112, DPTX_PHY_T6020, + DPTX_PHY_T8122, }; @@ -57,6 +61,17 @@ struct apple_dptx_phy { // TODO: m1n1 port things to clean up u32 active_lanes; + bool activation_complete; }; +/* + * Bound the T8122 cold activation sequence while bringing up new SoCs. A + * partial sequence deliberately ACKs DCP but suppresses later lane/rate MMIO, + * making each register group independently testable on a direct Linux boot. + */ +static unsigned int t8122_activation_stage = 6; +module_param(t8122_activation_stage, uint, 0644); +MODULE_PARM_DESC(t8122_activation_stage, + "T8122 activation stage to execute (0-6, 6 is complete)"); + static inline void mask32(void __iomem *reg, u32 mask, u32 set) @@ -123,4 +138,94 @@ static int dptx_phy_set_active_lane_count(struct apple_dptx_phy *phy, u32 num_la } +static int dptx_phy_activate_t8122(struct apple_dptx_phy *phy, u32 dcp_index) +{ + u32 val; + + phy->activation_complete = false; + dev_info(phy->dev, "T8122 PHY activation: begin (DCP %u)\n", + dcp_index); + if (!t8122_activation_stage) + return 0; + + /* Firmware raises the private PHY gate immediately before this callback. */ + usleep_range(5000, 6000); + + readl(phy->regs.core + 0x10); + writel(dcp_index, phy->regs.core + 0x10); + dev_info(phy->dev, "T8122 PHY activation: DCP input selected\n"); + if (t8122_activation_stage == 1) + return 0; + + set32(phy->regs.core + 0x48, 0x010); + set32(phy->regs.core + 0x48, 0x020); + clear32(phy->regs.core + 0x48, 0x040); + set32(phy->regs.core + 0x48, 0x100); + set32(phy->regs.core + 0x48, 0x200); + clear32(phy->regs.core + 0x48, 0x400); + set32(phy->regs.core + 0x48, 0x001); + set32(phy->regs.core + 0x48, 0x002); + clear32(phy->regs.core + 0x48, 0x004); + dev_info(phy->dev, "T8122 PHY activation: core reset sequence complete\n"); + if (t8122_activation_stage == 2) + return 0; + + val = readl(phy->regs.dptx + 0x2014); + writel((val & ~0x3f0000) | (0x30 << 16), + phy->regs.dptx + 0x2014); + set32(phy->regs.dptx + 0x20b8, 0x010000); + clear32(phy->regs.dptx + 0x2220, 0x0000002); + set32(phy->regs.dptx + 0x222c, 0x000800); + set32(phy->regs.dptx + 0x222c, 0x000100); + set32(phy->regs.dptx + 0x2230, 0x0200000); + clear32(phy->regs.dptx + 0x2278, 0x08000000); + set32(phy->regs.dptx + 0x22a4, 0x0000001); + dev_info(phy->dev, "T8122 PHY activation: common registers configured\n"); + if (t8122_activation_stage == 3) + return 0; + + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) { + val = readl(phy->regs.dptx + loff + 0x40); + writel((val & ~0xe00000) | 0x200000, + phy->regs.dptx + loff + 0x40); + } + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + set32(phy->regs.dptx + loff + 0x40, 0x080000); + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + clear32(phy->regs.dptx + loff + 0x244, 0x10); + dev_info(phy->dev, "T8122 PHY activation: lane registers configured\n"); + if (t8122_activation_stage == 4) + return 0; + + set32(phy->regs.dptx + 0x2218, 0x001); + clear32(phy->regs.dptx + 0x2234, 0x00000001); + set32(phy->regs.dptx + 0x2200, 0x0002); + clear32(phy->regs.dptx + 0x1000, 0x00000002); + set32(phy->regs.dptx + 0x4004, 0x08); + + dptx_phy_set_active_lane_count(phy, 0); + dev_info(phy->dev, "T8122 PHY activation: lanes parked\n"); + if (t8122_activation_stage == 5) + return 0; + + set32(phy->regs.dptx + 0xd000, 0); + dev_info(phy->dev, "T8122 PHY activation: calibration start\n"); + set32(phy->regs.dptx + 0x4224, 0); + set32(phy->regs.dptx + 0x4224, 1); + set32(phy->regs.dptx + 0x4200, 0x3c0); + set32(phy->regs.dptx + 0x4200, 1); + readl(phy->regs.dptx + 0x4204); + clear32(phy->regs.dptx + 0x4200, 1); + readl(phy->regs.dptx + 0x4210); + writel(0x2b0000, phy->regs.dptx + 0x4228); + set32(phy->regs.dptx + 0x4228, 0x8000); + phy->activation_complete = true; + dev_info(phy->dev, "T8122 PHY activation: complete\n"); + + return 0; +} + static int dptx_phy_activate(struct apple_dptx_phy *phy, u32 dcp_index) { @@ -130,4 +235,6 @@ static int dptx_phy_activate(struct apple_dptx_phy *phy, u32 dcp_index) dev_dbg(phy->dev, "activate(dcp:%u)\n", dcp_index); + if (phy->hw.type == DPTX_PHY_T8122) + return dptx_phy_activate_t8122(phy, dcp_index); // MMIO: R.4 0x23c500010 (dptx-phy[1], offset 0x10) = 0x0 @@ -379,5 +486,8 @@ static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) // MMIO: R.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a0 // MMIO: W.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x1109020 - clear32(phy->regs.dptx + 0x2220, 0x0000080); + if (phy->hw.type == DPTX_PHY_T8122) + clear32(phy->regs.dptx + 0x222c, 0x0000002); + else + clear32(phy->regs.dptx + 0x2220, 0x0000080); // MMIO: R.4 0x23c5420b0 (dptx-phy[0], offset 0x20b0) = 0x1e0e01c2 @@ -385,5 +495,6 @@ static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) val_20b0 = readl(phy->regs.dptx + 0x20b0); /* TODO: what happens on dptx-phy */ - if (phy->hw.type == DPTX_PHY_T6020) + if (phy->hw.type == DPTX_PHY_T6020 || + phy->hw.type == DPTX_PHY_T8122) val_20b0 = (val_20b0 & ~0x3ff) | 0x2a3; writel(val_20b0, phy->regs.dptx + 0x20b0); @@ -393,5 +504,6 @@ static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) val_20b4 = readl(phy->regs.dptx + 0x20b4); /* TODO: what happens on dptx-phy */ - if (phy->hw.type == DPTX_PHY_T6020) + if (phy->hw.type == DPTX_PHY_T6020 || + phy->hw.type == DPTX_PHY_T8122) val_20b4 = (val_20b4 | 0x4000000) & ~0x0008000; writel(val_20b4, phy->regs.dptx + 0x20b4); @@ -401,5 +513,7 @@ static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) val_20b4 = readl(phy->regs.dptx + 0x20b4); /* TODO: what happens on dptx-phy */ - if (phy->hw.type == DPTX_PHY_T6020) + if (phy->hw.type == DPTX_PHY_T8122) + val_20b4 &= ~0x0000002; + else if (phy->hw.type == DPTX_PHY_T6020) val_20b4 = (val_20b4 | 0x0000001) & ~0x0000004; writel(val_20b4, phy->regs.dptx + 0x20b4); @@ -416,5 +530,6 @@ static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 /* TODO: unclear */ - if (phy->hw.type == DPTX_PHY_T6020) + if (phy->hw.type == DPTX_PHY_T6020 || + phy->hw.type == DPTX_PHY_T8122) set32(phy->regs.dptx + 0x20b8, 0x010000); else @@ -422,5 +537,6 @@ static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 - clear32(phy->regs.dptx + 0x20b8, 0x200000); + if (phy->hw.type != DPTX_PHY_T8122) + clear32(phy->regs.dptx + 0x20b8, 0x200000); // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 @@ -530,4 +646,7 @@ static int dptx_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) switch (mode) { case PHY_MODE_INVALID: + if (dptx_phy->hw.type == DPTX_PHY_T8122 && + !dptx_phy->activation_complete) + return 0; return dptx_phy_deactivate(dptx_phy); case PHY_MODE_DP: @@ -575,4 +694,8 @@ static int dptx_phy_configure(struct phy *phy, union phy_configure_opts *opts_) int ret = 0; + if (dptx_phy->hw.type == DPTX_PHY_T8122 && + !dptx_phy->activation_complete) + return 0; + if (opts->set_lanes) { mutex_lock(&dptx_phy->lock); @@ -624,4 +747,5 @@ static int dptx_phy_probe(struct platform_device *pdev) struct apple_dptx_phy *dptx_phy; struct device *dev = &pdev->dev; + int ret; dptx_phy = devm_kzalloc(dev, sizeof(*dptx_phy), GFP_KERNEL); @@ -636,4 +760,12 @@ static int dptx_phy_probe(struct platform_device *pdev) mutex_init(&dptx_phy->lock); + /* + * The generic PHY core only propagates runtime-PM references to the + * provider when runtime PM was enabled before devm_phy_create(). + */ + ret = devm_pm_runtime_enable(dev); + if (ret) + return dev_err_probe(dev, ret, "failed to enable runtime PM\n"); + dptx_phy->regs.core = devm_platform_ioremap_resource_byname(pdev, "core"); @@ -668,7 +800,12 @@ static const struct apple_dptx_phy_hw apple_dptx_hw_t8112 = { }; +static const struct apple_dptx_phy_hw apple_dptx_hw_t8122 = { + .type = DPTX_PHY_T8122, +}; + static const struct of_device_id dptx_phy_match[] = { { .compatible = "apple,t6020-dptx-phy", .data = &apple_dptx_hw_t6020 }, { .compatible = "apple,t8112-dptx-phy", .data = &apple_dptx_hw_t8112 }, + { .compatible = "apple,t8122-dptx-phy", .data = &apple_dptx_hw_t8122 }, {}, }; diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 1baa6de9449589d934b5f26b34ce2a9ac9986b6c..f7cd7135ea012967d63668a94e01417509468f0b 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -4,4 +4,5 @@ obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o apple-dockchannel-y = dockchannel.o + obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o apple-mailbox-y = mailbox.o diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 330015eea2ebe4e7e122bd63ce382b278649eb64..5194e9a3c3293b2cdeaf9be103e9d2dae00992a6 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -35,4 +35,5 @@ #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) #define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) +#define APPLE_ASC_MBOX_CONTROL_ENABLE BIT(0) #define APPLE_ASC_MBOX_A2I_CONTROL 0x110 @@ -252,4 +253,19 @@ int apple_mbox_start(struct apple_mbox *mbox) return ret; + /* + * A firmware handoff can leave the I2A FIFO enabled with stale read and + * write pointers from the previous RTKit client. T8132 DCP firmware does + * not consume a new INIT message until the AP has reinitialized its receive + * FIFO. Native clients reset only OUTBOX (I2A) here; resetting A2I would + * race the IOP-owned read pointer and could discard an AP message. + */ + if (of_property_read_bool(mbox->dev->of_node, + "apple,reset-recv-fifo-on-start")) { + writel_relaxed(APPLE_ASC_MBOX_CONTROL_EMPTY | + APPLE_ASC_MBOX_CONTROL_ENABLE, + mbox->regs + mbox->hw->i2a_control); + readl_relaxed(mbox->regs + mbox->hw->i2a_control); + } + /* * Only some variants of this mailbox HW provide interrupt control diff --git a/drivers/soc/apple/rtkit-crashlog.c b/drivers/soc/apple/rtkit-crashlog.c index 8319e365110b87f967793c768e338bea9c0abe12..b841136850ba358732fb44131d9eadf25c91e2d9 100644 --- a/drivers/soc/apple/rtkit-crashlog.c +++ b/drivers/soc/apple/rtkit-crashlog.c @@ -15,4 +15,7 @@ #define APPLE_RTKIT_CRASHLOG_TIME FOURCC('C', 't', 'i', 'm') #define APPLE_RTKIT_CRASHLOG_REGS FOURCC('C', 'r', 'g', '8') +#define APPLE_RTKIT_CRASHLOG_STACK FOURCC('C', 'c', 's', 't') +#define APPLE_RTKIT_CRASHLOG_ASYNC_ERROR FOURCC('C', 'a', 's', 'C') +#define APPLE_RTKIT_CRASHLOG_DCP_MAP FOURCC('C', 'c', 'd', 'p') /* For COMPILE_TEST on non-ARM64 architectures */ @@ -61,4 +64,27 @@ struct apple_rtkit_crashlog_regs { static_assert(sizeof(struct apple_rtkit_crashlog_regs) == 0x350); +struct apple_rtkit_crashlog_dcp_map { + u64 va; + u64 dva; + u32 flags; +} __packed; +static_assert(sizeof(struct apple_rtkit_crashlog_dcp_map) == 0x14); + +struct apple_rtkit_crashlog_stack { + u32 task; + u32 unknown; +} __packed; +static_assert(sizeof(struct apple_rtkit_crashlog_stack) == 0x8); + +struct apple_rtkit_crashlog_async_error { + u64 l2c_err_sts; + u64 l2c_err_adr; + u64 l2c_err_inf; + u64 lsu_err_sts; + u64 fed_err_sts; + u64 mmu_err_sts; +} __packed; +static_assert(sizeof(struct apple_rtkit_crashlog_async_error) == 0x30); + static void apple_rtkit_crashlog_dump_str(struct apple_rtkit *rtk, u8 *bfr, size_t size) @@ -184,7 +210,76 @@ static void apple_rtkit_crashlog_dump_regs(struct apple_rtkit *rtk, u8 *bfr, } -void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) +static void apple_rtkit_crashlog_dump_dcp_map(struct apple_rtkit *rtk, + u8 *bfr, size_t size) +{ + struct apple_rtkit_crashlog_dcp_map entry; + size_t offset; + + for (offset = 0; offset + sizeof(entry) <= size; + offset += sizeof(entry)) { + memcpy(&entry, bfr + offset, sizeof(entry)); + dev_warn(rtk->dev, + "RTKit: DCP map: VA 0x%016llx -> DVA 0x%016llx flags 0x%x", + entry.va, entry.dva, entry.flags); + } +} + +static void apple_rtkit_crashlog_dump_stack(struct apple_rtkit *rtk, u8 *bfr, + size_t size) { + struct apple_rtkit_crashlog_stack stack; size_t offset; + u64 address; + + if (size < sizeof(stack)) { + dev_warn(rtk->dev, "RTKit: Stack section too small: 0x%zx", + size); + return; + } + + memcpy(&stack, bfr, sizeof(stack)); + dev_warn(rtk->dev, + "RTKit: Call stack (task %u, unknown 0x%x):", + stack.task, stack.unknown); + + for (offset = sizeof(stack); offset + sizeof(address) <= size; + offset += sizeof(address)) { + memcpy(&address, bfr + offset, sizeof(address)); + if (!address) + break; + dev_warn(rtk->dev, "RTKit: 0x%016llx", address); + } +} + +static void apple_rtkit_crashlog_dump_async_error(struct apple_rtkit *rtk, + u8 *bfr, size_t size) +{ + struct apple_rtkit_crashlog_async_error error; + + if (size < sizeof(error)) { + dev_warn(rtk->dev, + "RTKit: Async error section too small: 0x%zx", size); + return; + } + + memcpy(&error, bfr, sizeof(error)); + dev_warn(rtk->dev, "RTKit: Async error info:"); + dev_warn(rtk->dev, "RTKit: L2C_ERR_STS = 0x%016llx", + error.l2c_err_sts); + dev_warn(rtk->dev, "RTKit: L2C_ERR_ADR = 0x%016llx", + error.l2c_err_adr); + dev_warn(rtk->dev, "RTKit: L2C_ERR_INF = 0x%016llx", + error.l2c_err_inf); + dev_warn(rtk->dev, "RTKit: LSU_ERR_STS = 0x%016llx", + error.lsu_err_sts); + dev_warn(rtk->dev, "RTKit: FED_ERR_STS = 0x%016llx", + error.fed_err_sts); + dev_warn(rtk->dev, "RTKit: MMU_ERR_STS = 0x%016llx", + error.mmu_err_sts); +} + +void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) +{ + size_t offset, payload_size; u32 section_fourcc, section_size; struct apple_rtkit_crashlog_header header; @@ -209,4 +304,11 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) memcpy(§ion_fourcc, bfr + offset, 4); memcpy(§ion_size, bfr + offset + 12, 4); + if (section_size < 16 || section_size > size - offset) { + dev_warn(rtk->dev, + "RTKit: Invalid crashlog section size 0x%x at 0x%zx", + section_size, offset); + return; + } + payload_size = section_size - 16; switch (section_fourcc) { @@ -216,21 +318,33 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) case APPLE_RTKIT_CRASHLOG_STR: apple_rtkit_crashlog_dump_str(rtk, bfr + offset + 16, - section_size); + payload_size); break; case APPLE_RTKIT_CRASHLOG_VERSION: apple_rtkit_crashlog_dump_version( - rtk, bfr + offset + 16, section_size); + rtk, bfr + offset + 16, payload_size); break; case APPLE_RTKIT_CRASHLOG_MBOX: apple_rtkit_crashlog_dump_mailbox( - rtk, bfr + offset + 16, section_size); + rtk, bfr + offset + 16, payload_size); break; case APPLE_RTKIT_CRASHLOG_TIME: apple_rtkit_crashlog_dump_time(rtk, bfr + offset + 16, - section_size); + payload_size); break; case APPLE_RTKIT_CRASHLOG_REGS: apple_rtkit_crashlog_dump_regs(rtk, bfr + offset + 16, - section_size); + payload_size); + break; + case APPLE_RTKIT_CRASHLOG_STACK: + apple_rtkit_crashlog_dump_stack(rtk, bfr + offset + 16, + payload_size); + break; + case APPLE_RTKIT_CRASHLOG_ASYNC_ERROR: + apple_rtkit_crashlog_dump_async_error( + rtk, bfr + offset + 16, payload_size); + break; + case APPLE_RTKIT_CRASHLOG_DCP_MAP: + apple_rtkit_crashlog_dump_dcp_map( + rtk, bfr + offset + 16, payload_size); break; default: diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index c82065a8bf7b0301167f142e928007a77fca9f90..c6d0905dbd2e9a4283c6150971bcf76429e992cb 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -39,4 +39,7 @@ struct apple_rtkit { unsigned int iop_power_state; unsigned int ap_power_state; + bool boot_ap_power; + bool early_crashlog; + struct completion crashlog_completion; bool crashed; @@ -44,4 +47,5 @@ struct apple_rtkit { struct apple_rtkit_shmem ioreport_buffer; + struct apple_rtkit_shmem system7_buffer; struct apple_rtkit_shmem crashlog_buffer; struct apple_rtkit_shmem oslog_buffer; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index ef0c399ea85230d35656a655283fee339242f4ec..9bc4e1d7d6ecc4f411cbebeed2e23511862b2a7c 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -22,4 +22,6 @@ enum { APPLE_RTKIT_EP_DEBUG = 3, APPLE_RTKIT_EP_IOREPORT = 4, + /* Advertised by newer firmware and required to complete RTKit boot. */ + APPLE_RTKIT_EP_UNKNOWN7 = 7, APPLE_RTKIT_EP_OSLOG = 8, APPLE_RTKIT_EP_TRACEKIT = 0xa, @@ -62,4 +64,6 @@ enum { #define APPLE_RTKIT_SYSLOG_TYPE GENMASK_ULL(59, 52) +#define APPLE_RTKIT_SYSTEM7_BUFFER_REPLY 5 + #define APPLE_RTKIT_SYSLOG_LOG 5 @@ -168,4 +172,6 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) dev_dbg(rtk->dev, "RTKit: Discovered endpoint 0x%02x\n", ep); set_bit(ep, rtk->endpoints); + if (rtk->early_crashlog && ep == APPLE_RTKIT_EP_CRASHLOG) + apple_rtkit_start_ep(rtk, ep); } @@ -182,4 +188,6 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) { + if (rtk->early_crashlog && ep == APPLE_RTKIT_EP_CRASHLOG) + continue; switch (ep) { /* the management endpoint is started by default */ @@ -192,4 +200,5 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_DEBUG: case APPLE_RTKIT_EP_IOREPORT: + case APPLE_RTKIT_EP_UNKNOWN7: case APPLE_RTKIT_EP_OSLOG: case APPLE_RTKIT_EP_TRACEKIT: @@ -310,4 +319,6 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, } else { reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE, + ep == APPLE_RTKIT_EP_UNKNOWN7 ? + APPLE_RTKIT_SYSTEM7_BUFFER_REPLY : APPLE_RTKIT_BUFFER_REQUEST); reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, @@ -317,4 +328,6 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, } apple_rtkit_send_message(rtk, ep, reply, NULL, false); + } else if (rtk->early_crashlog && ep == APPLE_RTKIT_EP_CRASHLOG) { + apple_rtkit_send_message(rtk, ep, msg, NULL, false); } @@ -373,6 +386,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) if (!rtk->crashlog_buffer.size) { - apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, - APPLE_RTKIT_EP_CRASHLOG, msg); + if (!apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, + APPLE_RTKIT_EP_CRASHLOG, msg)) + complete_all(&rtk->crashlog_completion); return; } @@ -421,4 +435,19 @@ static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) } +static void apple_rtkit_system7_rx(struct apple_rtkit *rtk, u64 msg) +{ + u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); + + switch (type) { + case APPLE_RTKIT_BUFFER_REQUEST: + apple_rtkit_common_rx_get_buffer(rtk, &rtk->system7_buffer, + APPLE_RTKIT_EP_UNKNOWN7, msg); + break; + default: + dev_warn(rtk->dev, "RTKit: Unknown system endpoint 7 message: %llx\n", + msg); + } +} + static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg) { @@ -545,4 +574,7 @@ static void apple_rtkit_rx_work(struct work_struct *work) apple_rtkit_ioreport_rx(rtk, rtk_work->msg); break; + case APPLE_RTKIT_EP_UNKNOWN7: + apple_rtkit_system7_rx(rtk, rtk_work->msg); + break; case APPLE_RTKIT_EP_OSLOG: apple_rtkit_oslog_rx(rtk, rtk_work->msg); @@ -681,4 +713,6 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, rtk->cookie = cookie; rtk->ops = ops; + rtk->boot_ap_power = true; + init_completion(&rtk->crashlog_completion); init_completion(&rtk->epmap_completion); @@ -744,4 +778,5 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); + apple_rtkit_free_buffer(rtk, &rtk->system7_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer); @@ -758,4 +793,5 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) reinit_completion(&rtk->epmap_completion); + reinit_completion(&rtk->crashlog_completion); reinit_completion(&rtk->iop_pwr_ack_completion); reinit_completion(&rtk->ap_pwr_ack_completion); @@ -792,4 +828,112 @@ static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk, } +void apple_rtkit_set_boot_ap_power(struct apple_rtkit *rtk, bool enable) +{ + rtk->boot_ap_power = enable; +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_boot_ap_power); + +void apple_rtkit_set_early_crashlog(struct apple_rtkit *rtk) +{ + rtk->early_crashlog = true; + reinit_completion(&rtk->crashlog_completion); +} +EXPORT_SYMBOL_GPL(apple_rtkit_set_early_crashlog); + +int apple_rtkit_adopt_running(struct apple_rtkit *rtk, const u8 *endpoints, + size_t n_endpoints) +{ + size_t i; + + if (!rtk || (!endpoints && n_endpoints) || rtk->crashed) + return -EINVAL; + + bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); + set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); + for (i = 0; i < n_endpoints; i++) + set_bit(endpoints[i], rtk->endpoints); + + /* The bootloader already negotiated the current protocol and left both + * halves of the link live. No HELLO, endpoint map, INIT, or AP-power + * message may be replayed into that session. */ + rtk->version = APPLE_RTKIT_MAX_SUPPORTED_VERSION; + rtk->boot_result = 0; + rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_ON; + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; + complete_all(&rtk->epmap_completion); + complete_all(&rtk->iop_pwr_ack_completion); + complete_all(&rtk->ap_pwr_ack_completion); + + dev_info(rtk->dev, "RTKit: adopted running bootloader session\n"); + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_adopt_running); + +int apple_rtkit_reuse_syslog_buffer(struct apple_rtkit *rtk, dma_addr_t iova, + size_t size, size_t n_entries, + size_t msg_size) +{ + int ret; + + if (!rtk || !iova || !size || !n_entries || !msg_size) + return -EINVAL; + if (rtk->syslog_buffer.size || rtk->syslog_msg_buffer) + return -EBUSY; + if (!rtk->ops->shmem_setup) + return -EOPNOTSUPP; + + rtk->syslog_buffer.iova = iova; + rtk->syslog_buffer.size = size; + ret = rtk->ops->shmem_setup(rtk->cookie, &rtk->syslog_buffer); + if (ret) + goto clear_buffer; + + rtk->syslog_msg_buffer = kzalloc(msg_size, GFP_KERNEL); + if (!rtk->syslog_msg_buffer) { + ret = -ENOMEM; + goto destroy_buffer; + } + + rtk->syslog_n_entries = n_entries; + rtk->syslog_msg_size = msg_size; + dev_info(rtk->dev, + "RTKit: reused syslog buffer at %pad: entries: %zd, msg_size: %zd\n", + &rtk->syslog_buffer.iova, n_entries, msg_size); + return 0; + +destroy_buffer: + rtk->ops->shmem_destroy(rtk->cookie, &rtk->syslog_buffer); +clear_buffer: + rtk->syslog_buffer = (struct apple_rtkit_shmem){}; + return ret; +} +EXPORT_SYMBOL_GPL(apple_rtkit_reuse_syslog_buffer); + +int apple_rtkit_reuse_crashlog_buffer(struct apple_rtkit *rtk, + dma_addr_t iova, size_t size) +{ + int ret; + + if (!rtk || !iova || !size) + return -EINVAL; + if (rtk->crashlog_buffer.size) + return -EBUSY; + if (!rtk->ops->shmem_setup) + return -EOPNOTSUPP; + + rtk->crashlog_buffer.iova = iova; + rtk->crashlog_buffer.size = size; + ret = rtk->ops->shmem_setup(rtk->cookie, &rtk->crashlog_buffer); + if (ret) { + rtk->crashlog_buffer = (struct apple_rtkit_shmem){}; + return ret; + } + + dev_info(rtk->dev, "RTKit: reused crashlog buffer at %pad\n", + &rtk->crashlog_buffer.iova); + return 0; +} +EXPORT_SYMBOL_GPL(apple_rtkit_reuse_crashlog_buffer); + static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, unsigned int state) @@ -826,6 +970,8 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) dev_dbg(rtk->dev, "RTKit: waiting for boot to finish\n"); ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion); - if (ret) + if (ret) { + dev_err(rtk->dev, "RTKit: timed out waiting for endpoint map: %d\n", ret); return ret; + } if (rtk->boot_result) return rtk->boot_result; @@ -833,8 +979,29 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n"); ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion); - if (ret) + if (ret) { + dev_err(rtk->dev, + "RTKit: timed out waiting for IOP power ACK (state %#x): %d\n", + rtk->iop_power_state, ret); return ret; + } + if (rtk->early_crashlog) { + ret = apple_rtkit_wait_for_completion(&rtk->crashlog_completion); + if (ret) { + dev_err(rtk->dev, "RTKit: timed out waiting for crash buffer: %d\n", ret); + return ret; + } + } + if (!rtk->boot_ap_power) { + dev_dbg(rtk->dev, "RTKit: reusing boot AP power ownership\n"); + rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_ON; + return 0; + } - return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); + if (ret) + dev_err(rtk->dev, + "RTKit: failed waiting for AP power ACK (state %#x): %d\n", + rtk->ap_power_state, ret); + return ret; } EXPORT_SYMBOL_GPL(apple_rtkit_boot); @@ -948,4 +1115,5 @@ void apple_rtkit_free(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); + apple_rtkit_free_buffer(rtk, &rtk->system7_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer); diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index ec284aceb9093a10998a8654df74099ec0bdc282..be18f746b9f30e6a8f726889abb01107c2188f0a 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -98,4 +98,12 @@ config SERIAL_EARLYCON_RISCV_SBI enabled when early_param is processed. +config SERIAL_APPLE_DOCKCHANNEL_EARLYCON + bool "Earlycon on Apple DockChannel UART" + depends on ARCH_APPLE || COMPILE_TEST + select SERIAL_EARLYCON + default ARCH_APPLE + help + Support for early console output on the Apple DockChannel UART. + config SERIAL_SB1250_DUART tristate "BCM1xxx on-chip DUART serial support" diff --git a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile index bba7b21a4a1d32b36808b2b457b2dc56318e7ed2..779170882ccccfa9bee3603bf015beb9a1437d9a 100644 --- a/drivers/tty/serial/Makefile +++ b/drivers/tty/serial/Makefile @@ -10,4 +10,5 @@ obj-$(CONFIG_SERIAL_EARLYCON) += earlycon.o obj-$(CONFIG_SERIAL_EARLYCON_SEMIHOST) += earlycon-semihost.o obj-$(CONFIG_SERIAL_EARLYCON_RISCV_SBI) += earlycon-riscv-sbi.o +obj-$(CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON) += apple_dockchannel_early_uart.o # These Sparc drivers have to appear before others such as 8250 diff --git a/drivers/tty/serial/apple_dockchannel_early_uart.c b/drivers/tty/serial/apple_dockchannel_early_uart.c new file mode 100644 index 0000000000000000000000000000000000000000..b0ddde2e28432e13cb1f7ef64497fcff3aa82bb6 --- /dev/null +++ b/drivers/tty/serial/apple_dockchannel_early_uart.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel UART early console + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include + +#define DOCKCHANNEL_TX8 0x04 +#define DOCKCHANNEL_TX_FREE 0x14 + +static void dockchannel_uart_putc(struct uart_port *port, unsigned char c) +{ + while (readl(port->membase + DOCKCHANNEL_TX_FREE) == 0) + cpu_relax(); + writel(c, port->membase + DOCKCHANNEL_TX8); +} + +static void dockchannel_uart_early_write(struct console *con, const char *s, + unsigned int n) +{ + struct earlycon_device *dev = con->data; + + uart_console_write(&dev->port, s, n, dockchannel_uart_putc); +} + +static int __init dockchannel_uart_early_setup(struct earlycon_device *device, + const char *opt) +{ + if (!device->port.mapbase) + return -ENODEV; + + device->con->write = dockchannel_uart_early_write; + return 0; +} + +EARLYCON_DECLARE(dockchannel, dockchannel_uart_early_setup); diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 40c3ccfddb67027c440e9c8fa528a2599741c83f..569878adff190da849ef9c6d6c1d914e0cb0702f 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -437,4 +437,5 @@ static int dwc3_apple_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct dwc3_apple *appledwc; + enum usb_dr_mode dr_mode; int ret; @@ -445,4 +446,6 @@ static int dwc3_apple_probe(struct platform_device *pdev) appledwc->dev = &pdev->dev; mutex_init(&appledwc->lock); + dr_mode = usb_get_dr_mode(dev); + appledwc->dwc.dr_mode = dr_mode; appledwc->reset = devm_reset_control_get_exclusive(dev, NULL); @@ -476,4 +479,18 @@ static int dwc3_apple_probe(struct platform_device *pdev) */ appledwc->state = DWC3_APPLE_PROBE_PENDING; + if (dr_mode == USB_DR_MODE_HOST) { + /* + * Host-only instances have no cable event to start the core. Mark + * the role before core probe so dwc3_core_soft_reset() does not try + * to reset the unused gadget side of the controller. + */ + appledwc->dwc.current_dr_role = DWC3_GCTL_PRTCAP_HOST; + guard(mutex)(&appledwc->lock); + ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); + if (ret) + return dev_err_probe(dev, ret, "Failed to start fixed host controller\n"); + return 0; + } + ret = dwc3_apple_setup_role_switch(appledwc); if (ret) @@ -490,5 +507,6 @@ static void dwc3_apple_remove(struct platform_device *pdev) guard(mutex)(&appledwc->lock); - usb_role_switch_unregister(appledwc->role_sw); + if (appledwc->role_sw) + usb_role_switch_unregister(appledwc->role_sw); /* diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index bfe616194dfad407dd6ab0f5d3a3eea529cee4ee..cf7f36be378dd48845c164c2dae0b9464a3a6c7e 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -847,8 +847,9 @@ static void dwc3_ep0_inspect_setup(struct dwc3 *dwc, } - if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) + if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) { ret = dwc3_ep0_std_request(dwc, ctrl); - else + } else { ret = dwc3_ep0_delegate_req(dwc, ctrl); + } if (ret == USB_GADGET_DELAYED_STATUS) @@ -954,4 +955,5 @@ static void dwc3_ep0_complete_status(struct dwc3 *dwc, dwc->ep0state = EP0_SETUP_PHASE; + dwc3_ep0_out_start(dwc); } diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index 8a6a1c663eb699c2b65a7fe915eaaada2b52a6ab..eebe658de6f8c89d31e89b8897f7a36ebacb38b0 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -6,5 +6,5 @@ obj-$(CONFIG_TYPEC) += altmodes/ obj-$(CONFIG_TYPEC_TCPM) += tcpm/ obj-$(CONFIG_TYPEC_UCSI) += ucsi/ -obj-$(CONFIG_TYPEC_TPS6598X) += tipd/ +obj-$(CONFIG_TYPEC_TPS6598X_CORE) += tipd/ obj-$(CONFIG_TYPEC_ANX7411) += anx7411.o obj-$(CONFIG_TYPEC_HD3SS3220) += hd3ss3220.o diff --git a/include/linux/pci-pwrctrl.h b/include/linux/pci-pwrctrl.h index 1192a2527521d1db8a116f532f70017a0336ba1f..f27be6489b50fd03c1783982ae139b9c964ca02d 100644 --- a/include/linux/pci-pwrctrl.h +++ b/include/linux/pci-pwrctrl.h @@ -60,4 +60,8 @@ void pci_pwrctrl_destroy_devices(struct device *parent); int pci_pwrctrl_power_on_devices(struct device *parent); void pci_pwrctrl_power_off_devices(struct device *parent); +int pci_pwrctrl_create_device(struct device_node *np, struct device *parent); +void pci_pwrctrl_destroy_device(struct device_node *np); +int pci_pwrctrl_power_on_device(struct device_node *np); +void pci_pwrctrl_power_off_device(struct device_node *np); #else static inline int pci_pwrctrl_create_devices(struct device *parent) { return 0; } diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 42a0666e76fcc5cca5fbf2d51387d8c1e080b7e1..1b0b7cd0bc69b8e014784cf0e217290f524d03dc 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -106,4 +106,8 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, void apple_rtkit_free(struct apple_rtkit *rtk); +/* Configure before starting the coprocessor. G16 requests its preallocated + * crash buffer before the endpoint-map ACK and requires an explicit reply. */ +void apple_rtkit_set_early_crashlog(struct apple_rtkit *rtk); + /* * Reinitialize internal structures. Must only be called with the co-processor @@ -112,4 +116,29 @@ void apple_rtkit_free(struct apple_rtkit *rtk); int apple_rtkit_reinit(struct apple_rtkit *rtk); +/* + * Select whether apple_rtkit_boot() should request AP power ownership. Some + * bootloader handoffs retain that ownership for the next RTKit client. + */ +void apple_rtkit_set_boot_ap_power(struct apple_rtkit *rtk, bool enable); + +/* + * Adopt a bootloader RTKit session which remains in the ON state. The caller + * supplies the endpoint map because a running session does not replay it. + */ +int apple_rtkit_adopt_running(struct apple_rtkit *rtk, const u8 *endpoints, + size_t n_endpoints); + +/* + * Restore a syslog buffer retained across a restartable bootloader handoff. + * The buffer must still be mapped in the coprocessor's IOMMU domain. + */ +int apple_rtkit_reuse_syslog_buffer(struct apple_rtkit *rtk, dma_addr_t iova, + size_t size, size_t n_entries, + size_t msg_size); + +/* Restore the crashlog buffer belonging to an adopted RTKit session. */ +int apple_rtkit_reuse_crashlog_buffer(struct apple_rtkit *rtk, + dma_addr_t iova, size_t size); + /* * Handle RTKit's boot process. Should be called after the CPU of the diff --git a/rust/helpers/dma-fence.c b/rust/helpers/dma-fence.c index 6491016262934b814408f1f6a83bd6ddc6acd0c3..c968d09399915f90e0fb6f26575324845e0ff257 100644 --- a/rust/helpers/dma-fence.c +++ b/rust/helpers/dma-fence.c @@ -31,3 +31,8 @@ void rust_helper_dma_fence_set_error(struct dma_fence *fence, int error) } +bool rust_helper_dma_fence_is_signaled(struct dma_fence *fence) +{ + return dma_fence_is_signaled(fence); +} + #endif diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 3603e473c4e0f63446ebb8798dceff3d85b3a2bd..7ac519d6af41d9a45302563c7cffb7156ce91062 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -21,4 +21,8 @@ pub trait DriverFile { fn open(device: &drm::Device) -> Result>>; + /// Called once after associating the driver data with the DRM file, before + /// making the file available to userspace. + fn post_open(&self, _file: &File) where Self: Sized {} + /// Get raw drm_file pointer fn as_raw(&self) -> *mut bindings::drm_file; @@ -45,5 +49,6 @@ impl File { } - pub(super) fn as_raw(&self) -> *mut bindings::drm_file { + /// Return the DRM core pointer, valid for this open file's lifetime. + pub fn as_raw(&self) -> *mut bindings::drm_file { self.0.get() } @@ -88,4 +93,6 @@ impl File { unsafe { (*file.as_raw()).driver_priv = driver_priv.cast() }; + file.inner().post_open(file); + 0 } diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index b3e79f83c60020d4d3b16bb03568bebd1321b122..29a02a77c5d7bf1a74a8c36649886768f93550a2 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -337,6 +337,8 @@ impl driver::AllocImpl for Object { prime_fd_to_handle: None, gem_prime_import: None, - gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), - dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + // The generic C allocators do not initialize the Rust object payload. + // Support native GEM allocation and export only. + gem_prime_import_sg_table: None, + dumb_create: None, dumb_map_offset: None, }; @@ -462,2 +464,7 @@ impl Deref for SGTable { } } + +impl SGTable { + /// Return the GEM object that owns and pins this scatter-gather table. + pub fn object(&self) -> &Object { &self._owner } +} diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 599538dab9a8b00a2b8d0c5cb2b6fd130927c897..0d2fe3818c2d9651102e46568f14f030b03be5c4 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -392,4 +392,25 @@ impl Mem { self.size } + + /// Borrows a bounded byte range as ordinary RAM for IO-agnostic APIs. + pub fn as_iosys_map( + &mut self, + offset: usize, + size: usize, + ) -> Result> { + use crate::iosys_map::{IoSysMapRef, RawIoSysMap}; + if offset.checked_add(size).ok_or(EINVAL)? > self.size { + return Err(EINVAL); + } + let raw = bindings::iosys_map { + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { + // SAFETY: The checked range lies inside this owned mapping. + vaddr: unsafe { self.ptr().add(offset) }.cast(), + }, + is_iomem: false, + }; + // SAFETY: The map is valid for the duration of this mutable borrow. + Ok(unsafe { IoSysMapRef::new(RawIoSysMap::from_raw(raw), size) }) + } } diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index a5bfcfb8fa733434c1bcb0ad2b7eb4fd8cfd0199..a5ca5819769f8c27fcbf27b53c6e5b8f31ca638a 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -243,4 +243,11 @@ impl RtKit { } + /// Selects the early preallocated crash-buffer exchange used by G16. + /// Call before starting the coprocessor CPU. + pub fn set_early_crashlog(self: Pin<&mut Self>) { + // SAFETY: The RTKit handle is valid by the type invariant. + unsafe { bindings::apple_rtkit_set_early_crashlog(self.rtk) }; + } + /// Boots (wakes up) the RTKit coprocessor. pub fn wake(self: Pin<&mut Self>) -> Result { @@ -255,4 +262,11 @@ impl RtKit { } + /// Quiesces the host interface and puts the coprocessor to sleep. + /// Callers must retain firmware-accessible memory if this fails. + pub fn shutdown(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_shutdown(self.rtk) }) + } + /// Starts a non-system endpoint. pub fn start_endpoint(self: Pin<&mut Self>, endpoint: u8) -> Result {