diff options
Diffstat (limited to 'drivers/gpu/drm/xe')
185 files changed, 4776 insertions, 1903 deletions
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 03242e8b3d87..8e7b146880f4 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -16,14 +16,14 @@ subdir-ccflags-y += -I$(obj) -I$(src) hostprogs := xe_gen_wa_oob generated_oob := $(obj)/generated/xe_wa_oob.c $(obj)/generated/xe_wa_oob.h quiet_cmd_wa_oob = GEN $(notdir $(generated_oob)) - cmd_wa_oob = mkdir -p $(@D); $^ $(generated_oob) + cmd_wa_oob = mkdir -p $(@D); $(obj)/xe_gen_wa_oob $(src)/xe_wa_oob.rules $(generated_oob) $(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \ $(src)/xe_wa_oob.rules $(call cmd,wa_oob) generated_device_oob := $(obj)/generated/xe_device_wa_oob.c $(obj)/generated/xe_device_wa_oob.h quiet_cmd_device_wa_oob = GEN $(notdir $(generated_device_oob)) - cmd_device_wa_oob = mkdir -p $(@D); $^ $(generated_device_oob) + cmd_device_wa_oob = mkdir -p $(@D); $(obj)/xe_gen_wa_oob $(src)/xe_device_wa_oob.rules $(generated_device_oob) $(obj)/generated/%_device_wa_oob.c $(obj)/generated/%_device_wa_oob.h: $(obj)/xe_gen_wa_oob \ $(src)/xe_device_wa_oob.rules $(call cmd,device_wa_oob) @@ -113,6 +113,7 @@ xe-y += xe_bb.o \ xe_pxp_submit.o \ xe_query.o \ xe_range_fence.o \ + xe_ras.o \ xe_reg_sr.o \ xe_reg_whitelist.o \ xe_ring_ops.o \ @@ -124,6 +125,9 @@ xe-y += xe_bb.o \ xe_step.o \ xe_survivability_mode.o \ xe_sync.o \ + xe_sysctrl.o \ + xe_sysctrl_event.o \ + xe_sysctrl_mailbox.o \ xe_tile.o \ xe_tile_sysfs.o \ xe_tlb_inval.o \ @@ -212,7 +216,6 @@ $(obj)/i915-display/%.o: $(srctree)/drivers/gpu/drm/i915/display/%.c FORCE # Display code specific to xe xe-$(CONFIG_DRM_XE_DISPLAY) += \ - display/intel_fbdev_fb.o \ display/xe_display.o \ display/xe_display_bo.o \ display/xe_display_pcode.o \ @@ -262,6 +265,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/intel_display_power.o \ i915-display/intel_display_power_map.o \ i915-display/intel_display_power_well.o \ + i915-display/intel_display_reset.o \ i915-display/intel_display_rpm.o \ i915-display/intel_display_rps.o \ i915-display/intel_display_trace.o \ @@ -305,6 +309,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/intel_link_bw.o \ i915-display/intel_lspcon.o \ i915-display/intel_lt_phy.o \ + i915-display/intel_mchbar.o \ i915-display/intel_modeset_lock.o \ i915-display/intel_modeset_setup.o \ i915-display/intel_modeset_verify.o \ @@ -331,7 +336,8 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/skl_prefill.o \ i915-display/skl_scaler.o \ i915-display/skl_universal_plane.o \ - i915-display/skl_watermark.o + i915-display/skl_watermark.o \ + i915-display/vlv_sideband.o ifeq ($(CONFIG_ACPI),y) xe-$(CONFIG_DRM_XE_DISPLAY) += \ diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h index e33bd622ab44..644f5a4226d7 100644 --- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h @@ -192,6 +192,13 @@ enum { * `GuC KLV`_ keys available for use with PF2GUC_UPDATE_VGT_POLICY. * * _`GUC_KLV_VGT_POLICY_SCHED_IF_IDLE` : 0x8001 + * [From 70.12.0] + * This config allows to update scheduling priority of PF and all VFs at once. + * Setting this policy to 0 updates all VFs scheduling priorities to LOW, and + * setting this policy to 1 updates all VFs scheduling priorities to NORMAL. + * Those changes will take effect on the next VF-Switch event. + * + * [Before 70.12.0] * This config sets whether strict scheduling is enabled whereby any VF * that doesn’t have work to submit is still allocated a fixed execution * time-slice to ensure active VFs execution is always consistent even @@ -496,6 +503,7 @@ enum xe_guc_klv_ids { GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO = 0x900a, GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH = 0x900b, GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG = 0x900c, + GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG = 0x900e, }; #endif diff --git a/drivers/gpu/drm/xe/xe_uc_fw_abi.h b/drivers/gpu/drm/xe/abi/uc_fw_abi.h index 3c9a63d13032..198e949660e0 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw_abi.h +++ b/drivers/gpu/drm/xe/abi/uc_fw_abi.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_UC_FW_ABI_H -#define _XE_UC_FW_ABI_H +#ifndef _ABI_UC_FW_ABI_H +#define _ABI_UC_FW_ABI_H #include <linux/build_bug.h> #include <linux/types.h> diff --git a/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h b/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h new file mode 100644 index 000000000000..4cbde267ac44 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_ABI_H_ +#define _XE_SYSCTRL_ABI_H_ + +#include <linux/types.h> + +/** + * DOC: System Controller ABI + * + * This header defines the Application Binary Interface (ABI) used by + * drm/xe to communicate with System Controller firmware on Intel Xe3p + * discrete GPU platforms. + * + * System Controller (sysctrl) is a firmware-managed entity on Intel + * dGPUs responsible for certain low-level platform management + * functions. + * + * Communication protocol: + * + * Communication uses a mailbox interface with messages composed of: + * + * - Application message header (struct xe_sysctrl_app_msg_hdr) + * containing group_id, command, and version + * - Variable-length, command-specific payload + * + * Message header format: + * + * The 32-bit application message header is packed as: + * + * - Bits [7:0] : Group ID identifying command group + * - Bits [15:8] : Command identifier within group + * - Bits [23:16] : Command version for interface compatibility + * - Bits [31:24] : Reserved, must be zero + * + * This header defines firmware ABI message formats and constants shared + * between driver and System Controller firmware. + */ + +/** + * struct xe_sysctrl_app_msg_hdr - Application layer message header + * @data: 32-bit header data + * + * Header structure for application-level messages. + */ +struct xe_sysctrl_app_msg_hdr { + u32 data; +} __packed; + +#define SYSCTRL_HDR_GROUP_ID_MASK GENMASK(7, 0) +#define SYSCTRL_HDR_COMMAND_MASK GENMASK(14, 8) +#define SYSCTRL_HDR_COMMAND_MAX 0x7f +#define SYSCTRL_HDR_IS_RESPONSE BIT(15) +#define SYSCTRL_HDR_RESERVED_MASK GENMASK(23, 16) +#define SYSCTRL_HDR_RESULT_MASK GENMASK(31, 24) + +#define APP_HDR_GROUP_ID_MASK GENMASK(7, 0) +#define APP_HDR_COMMAND_MASK GENMASK(15, 8) +#define APP_HDR_VERSION_MASK GENMASK(23, 16) +#define APP_HDR_RESERVED_MASK GENMASK(31, 24) + +#endif diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h index ce986f0e8f38..552975a30ba2 100644 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h +++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h @@ -3,4 +3,12 @@ * Copyright © 2023 Intel Corporation */ -#include "../../i915/intel_clock_gating.h" +#ifndef __INTEL_CLOCK_GATING_H__ +#define __INTEL_CLOCK_GATING_H__ + +struct drm_device; + +static inline void intel_clock_gating_init(struct drm_device *drm) {} +static inline void intel_clock_gating_hooks_init(struct drm_device *drm) {} + +#endif /* __INTEL_CLOCK_GATING_H__ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h deleted file mode 100644 index 55b316985340..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h +++ /dev/null @@ -1,6 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#include "../../i915/intel_mchbar_regs.h" diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h deleted file mode 100644 index 8c15867fd613..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h +++ /dev/null @@ -1,6 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#include "../../i915/intel_pci_config.h" diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h deleted file mode 100644 index 0eabe2866f5f..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#ifndef __INTEL_STEP_H__ -#define __INTEL_STEP_H__ - -#include "xe_step_types.h" - -#define intel_step xe_step - -#endif /* __INTEL_STEP_H__ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h b/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h deleted file mode 100644 index 69e1935e9cdf..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2013-2021 Intel Corporation - */ - -#ifndef _VLV_IOSF_SB_H_ -#define _VLV_IOSF_SB_H_ - -#include <linux/types.h> - -#include "vlv_iosf_sb_reg.h" - -struct drm_device; - -enum vlv_iosf_sb_unit { - VLV_IOSF_SB_BUNIT, - VLV_IOSF_SB_CCK, - VLV_IOSF_SB_CCU, - VLV_IOSF_SB_DPIO, - VLV_IOSF_SB_DPIO_2, - VLV_IOSF_SB_FLISDSI, - VLV_IOSF_SB_GPIO, - VLV_IOSF_SB_NC, - VLV_IOSF_SB_PUNIT, -}; - -static inline void vlv_iosf_sb_get(struct drm_device *drm, unsigned long ports) -{ -} -static inline u32 vlv_iosf_sb_read(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr) -{ - return 0; -} -static inline int vlv_iosf_sb_write(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr, u32 val) -{ - return 0; -} -static inline void vlv_iosf_sb_put(struct drm_device *drm, unsigned long ports) -{ -} - -#endif /* _VLV_IOSF_SB_H_ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h b/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h deleted file mode 100644 index cb7fa8e794a6..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h +++ /dev/null @@ -1,6 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#include "../../i915/vlv_iosf_sb_reg.h" diff --git a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c deleted file mode 100644 index d7030e4d814c..000000000000 --- a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c +++ /dev/null @@ -1,120 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#include <linux/fb.h> - -#include "intel_fbdev_fb.h" -#include "xe_bo.h" -#include "xe_ttm_stolen_mgr.h" -#include "xe_wa.h" - -#include <generated/xe_device_wa_oob.h> - -/* - * FIXME: There shouldn't be any reason to have XE_PAGE_SIZE stride - * alignment. The same 64 as i915 uses should be fine, and we shouldn't need to - * have driver specific values. However, dropping the stride alignment to 64 - * leads to underflowing the bo pin count in the atomic cleanup work. - */ -u32 intel_fbdev_fb_pitch_align(u32 stride) -{ - return ALIGN(stride, XE_PAGE_SIZE); -} - -bool intel_fbdev_fb_prefer_stolen(struct drm_device *drm, unsigned int size) -{ - struct xe_device *xe = to_xe_device(drm); - struct ttm_resource_manager *stolen; - - stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); - if (!stolen) - return false; - - if (IS_DGFX(xe)) - return false; - - if (XE_DEVICE_WA(xe, 22019338487_display)) - return false; - - /* - * If the FB is too big, just don't use it since fbdev is not very - * important and we should probably use that space with FBC or other - * features. - */ - return stolen->size >= size * 2; -} - -struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size) -{ - struct xe_device *xe = to_xe_device(drm); - struct xe_bo *obj; - - obj = ERR_PTR(-ENODEV); - - if (intel_fbdev_fb_prefer_stolen(drm, size)) { - obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), - size, - ttm_bo_type_kernel, - XE_BO_FLAG_FORCE_WC | - XE_BO_FLAG_STOLEN | - XE_BO_FLAG_GGTT, - false); - if (!IS_ERR(obj)) - drm_info(&xe->drm, "Allocated fbdev into stolen\n"); - else - drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj)); - } else { - drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n"); - } - - if (IS_ERR(obj)) { - obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size, - ttm_bo_type_kernel, - XE_BO_FLAG_FORCE_WC | - XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | - XE_BO_FLAG_GGTT, - false); - } - - if (IS_ERR(obj)) { - drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj); - return ERR_PTR(-ENOMEM); - } - - return &obj->ttm.base; -} - -void intel_fbdev_fb_bo_destroy(struct drm_gem_object *obj) -{ - xe_bo_unpin_map_no_vm(gem_to_xe_bo(obj)); -} - -int intel_fbdev_fb_fill_info(struct drm_device *drm, struct fb_info *info, - struct drm_gem_object *_obj, struct i915_vma *vma) -{ - struct xe_bo *obj = gem_to_xe_bo(_obj); - struct pci_dev *pdev = to_pci_dev(drm->dev); - - if (!(obj->flags & XE_BO_FLAG_SYSTEM)) { - if (obj->flags & XE_BO_FLAG_STOLEN) - info->fix.smem_start = xe_ttm_stolen_io_offset(obj, 0); - else - info->fix.smem_start = - pci_resource_start(pdev, 2) + - xe_bo_addr(obj, 0, XE_PAGE_SIZE); - - info->fix.smem_len = obj->ttm.base.size; - } else { - /* XXX: Pure fiction, as the BO may not be physically accessible.. */ - info->fix.smem_start = 0; - info->fix.smem_len = obj->ttm.base.size; - } - XE_WARN_ON(iosys_map_is_null(&obj->vmap)); - - info->screen_base = obj->vmap.vaddr_iomem; - info->screen_size = obj->ttm.base.size; - - return 0; -} diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c index b17fb698d2f8..810d93fefcbc 100644 --- a/drivers/gpu/drm/xe/display/xe_display.c +++ b/drivers/gpu/drm/xe/display/xe_display.c @@ -36,10 +36,12 @@ #include "intel_hotplug.h" #include "intel_opregion.h" #include "skl_watermark.h" +#include "xe_device.h" #include "xe_display_bo.h" #include "xe_display_pcode.h" #include "xe_display_rpm.h" #include "xe_dsb_buffer.h" +#include "xe_fb_pin.h" #include "xe_frontbuffer.h" #include "xe_hdcp_gsc.h" #include "xe_initial_plane.h" @@ -69,31 +71,9 @@ bool xe_display_driver_probe_defer(struct pci_dev *pdev) return intel_display_driver_probe_defer(pdev); } -/** - * xe_display_driver_set_hooks - Add driver flags and hooks for display - * @driver: DRM device driver - * - * Set features and function hooks in @driver that are needed for driving the - * display IP. This sets the driver's capability of driving display, regardless - * if the device has it enabled - * - * Note: This is called before xe or display device creation. - */ -void xe_display_driver_set_hooks(struct drm_driver *driver) -{ - if (!xe_modparam.probe_display) - return; - -#ifdef CONFIG_DRM_FBDEV_EMULATION - driver->fbdev_probe = intel_fbdev_driver_fbdev_probe; -#endif - - driver->driver_features |= DRIVER_MODESET | DRIVER_ATOMIC; -} - static void unset_display_features(struct xe_device *xe) { - xe->drm.driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC); + xe->drm.driver_features &= ~XE_DISPLAY_DRIVER_FEATURES; } static void xe_display_fini_early(void *arg) @@ -108,7 +88,7 @@ static void xe_display_fini_early(void *arg) intel_display_driver_remove_nogem(display); intel_display_driver_remove_noirq(display); intel_opregion_cleanup(display); - intel_power_domains_cleanup(display); + intel_display_power_cleanup(display); } int xe_display_init_early(struct xe_device *xe) @@ -157,7 +137,7 @@ int xe_display_init_early(struct xe_device *xe) return devm_add_action_or_reset(xe->drm.dev, xe_display_fini_early, xe); err_noirq: intel_display_driver_remove_noirq(display); - intel_power_domains_cleanup(display); + intel_display_power_cleanup(display); err_opregion: intel_opregion_cleanup(display); return err; @@ -197,7 +177,7 @@ void xe_display_register(struct xe_device *xe) return; intel_display_driver_register(display); - intel_power_domains_enable(display); + intel_display_power_enable(display); } void xe_display_unregister(struct xe_device *xe) @@ -207,7 +187,7 @@ void xe_display_unregister(struct xe_device *xe) if (!xe->info.probe_display) return; - intel_power_domains_disable(display); + intel_display_power_disable(display); intel_display_driver_unregister(display); } @@ -221,7 +201,7 @@ void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl) return; if (master_ctl & DISPLAY_IRQ) - gen11_display_irq_handler(display); + intel_display_irq_handler(display, NULL); } void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir) @@ -242,7 +222,7 @@ void xe_display_irq_reset(struct xe_device *xe) if (!xe->info.probe_display) return; - gen11_display_irq_reset(display); + intel_display_irq_reset(display); } void xe_display_irq_postinstall(struct xe_device *xe) @@ -252,7 +232,7 @@ void xe_display_irq_postinstall(struct xe_device *xe) if (!xe->info.probe_display) return; - gen11_de_irq_postinstall(display); + intel_display_irq_postinstall(display); } static bool suspend_to_idle(void) @@ -264,27 +244,6 @@ static bool suspend_to_idle(void) return false; } -static void xe_display_flush_cleanup_work(struct xe_device *xe) -{ - struct intel_crtc *crtc; - - for_each_intel_crtc(&xe->drm, crtc) { - struct drm_crtc_commit *commit; - - spin_lock(&crtc->base.commit_lock); - commit = list_first_entry_or_null(&crtc->base.commit_list, - struct drm_crtc_commit, commit_entry); - if (commit) - drm_crtc_commit_get(commit); - spin_unlock(&crtc->base.commit_lock); - - if (commit) { - wait_for_completion(&commit->cleanup_done); - drm_crtc_commit_put(commit); - } - } -} - static void xe_display_enable_d3cold(struct xe_device *xe) { struct intel_display *display = xe->display; @@ -296,9 +255,9 @@ static void xe_display_enable_d3cold(struct xe_device *xe) * We do a lot of poking in a lot of registers, make sure they work * properly. */ - intel_power_domains_disable(display); + intel_display_power_disable(display); - xe_display_flush_cleanup_work(xe); + intel_display_flush_cleanup_work(display); intel_opregion_suspend(display, PCI_D3cold); @@ -329,7 +288,7 @@ static void xe_display_disable_d3cold(struct xe_device *xe) intel_opregion_resume(display); - intel_power_domains_enable(display); + intel_display_power_enable(display); } void xe_display_pm_suspend(struct xe_device *xe) @@ -344,7 +303,7 @@ void xe_display_pm_suspend(struct xe_device *xe) * We do a lot of poking in a lot of registers, make sure they work * properly. */ - intel_power_domains_disable(display); + intel_display_power_disable(display); drm_client_dev_suspend(&xe->drm); if (intel_display_device_present(display)) { @@ -353,7 +312,7 @@ void xe_display_pm_suspend(struct xe_device *xe) intel_display_driver_suspend(display); } - xe_display_flush_cleanup_work(xe); + intel_display_flush_cleanup_work(display); intel_encoder_block_all_hpds(display); @@ -376,7 +335,7 @@ void xe_display_pm_shutdown(struct xe_device *xe) if (!xe->info.probe_display) return; - intel_power_domains_disable(display); + intel_display_power_disable(display); drm_client_dev_suspend(&xe->drm); if (intel_display_device_present(display)) { @@ -385,7 +344,7 @@ void xe_display_pm_shutdown(struct xe_device *xe) intel_display_driver_suspend(display); } - xe_display_flush_cleanup_work(xe); + intel_display_flush_cleanup_work(display); intel_dp_mst_suspend(display); intel_encoder_block_all_hpds(display); intel_hpd_cancel_work(display); @@ -457,7 +416,7 @@ void xe_display_pm_shutdown_late(struct xe_device *xe) * for now leaving all display power wells in the INIT power domain * enabled. */ - intel_power_domains_driver_remove(display); + intel_display_power_driver_remove(display); } void xe_display_pm_resume_early(struct xe_device *xe) @@ -504,7 +463,7 @@ void xe_display_pm_resume(struct xe_device *xe) drm_client_dev_resume(&xe->drm); - intel_power_domains_enable(display); + intel_display_power_enable(display); } void xe_display_pm_runtime_resume(struct xe_device *xe) @@ -527,9 +486,10 @@ void xe_display_pm_runtime_resume(struct xe_device *xe) static void display_device_remove(struct drm_device *dev, void *arg) { - struct intel_display *display = arg; + struct xe_device *xe = arg; - intel_display_device_remove(display); + intel_display_device_remove(xe->display); + xe->display = NULL; } static bool irq_enabled(struct drm_device *drm) @@ -559,6 +519,7 @@ static bool has_auxccs(struct drm_device *drm) static const struct intel_display_parent_interface parent = { .bo = &xe_display_bo_interface, .dsb = &xe_display_dsb_interface, + .fb_pin = &xe_display_fb_pin_interface, .frontbuffer = &xe_display_frontbuffer_interface, .hdcp = &xe_display_hdcp_interface, .initial_plane = &xe_display_initial_plane_interface, @@ -594,12 +555,12 @@ int xe_display_probe(struct xe_device *xe) if (IS_ERR(display)) return PTR_ERR(display); - err = drmm_add_action_or_reset(&xe->drm, display_device_remove, display); + xe->display = display; + + err = drmm_add_action_or_reset(&xe->drm, display_device_remove, xe); if (err) return err; - xe->display = display; - if (intel_display_device_present(display)) return 0; @@ -608,3 +569,11 @@ no_display: unset_display_features(xe); return 0; } + +#ifdef CONFIG_DRM_FBDEV_EMULATION +int xe_display_driver_fbdev_probe(struct drm_fb_helper *fbh, + struct drm_fb_helper_surface_size *sizes) +{ + return intel_fbdev_driver_fbdev_probe(fbh, sizes); +} +#endif diff --git a/drivers/gpu/drm/xe/display/xe_display.h b/drivers/gpu/drm/xe/display/xe_display.h index 76db95c25f7e..60291cb154df 100644 --- a/drivers/gpu/drm/xe/display/xe_display.h +++ b/drivers/gpu/drm/xe/display/xe_display.h @@ -6,14 +6,20 @@ #ifndef _XE_DISPLAY_H_ #define _XE_DISPLAY_H_ -#include "xe_device.h" +#include <linux/types.h> struct drm_driver; +struct drm_fb_helper; +struct drm_fb_helper_surface_size; +struct pci_dev; +struct xe_device; #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) bool xe_display_driver_probe_defer(struct pci_dev *pdev); -void xe_display_driver_set_hooks(struct drm_driver *driver); + +int xe_display_driver_fbdev_probe(struct drm_fb_helper *fbh, + struct drm_fb_helper_surface_size *sizes); int xe_display_probe(struct xe_device *xe); @@ -38,11 +44,18 @@ void xe_display_pm_runtime_suspend(struct xe_device *xe); void xe_display_pm_runtime_suspend_late(struct xe_device *xe); void xe_display_pm_runtime_resume(struct xe_device *xe); +#define XE_DISPLAY_DRIVER_FEATURES (DRIVER_MODESET | DRIVER_ATOMIC) +#define XE_DISPLAY_DRIVER_OPS \ + .fbdev_probe = PTR_IF(IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION), \ + xe_display_driver_fbdev_probe) + #else +#define XE_DISPLAY_DRIVER_FEATURES 0 +#define XE_DISPLAY_DRIVER_OPS \ + .fbdev_probe = NULL + static inline int xe_display_driver_probe_defer(struct pci_dev *pdev) { return 0; } -static inline void xe_display_driver_set_hooks(struct drm_driver *driver) { } -static inline void xe_display_driver_remove(struct xe_device *xe) {} static inline int xe_display_probe(struct xe_device *xe) { return 0; } diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.c b/drivers/gpu/drm/xe/display/xe_display_bo.c index 1d81b9908265..8953da0136dc 100644 --- a/drivers/gpu/drm/xe/display/xe_display_bo.c +++ b/drivers/gpu/drm/xe/display/xe_display_bo.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: MIT /* Copyright © 2024 Intel Corporation */ +#include <linux/fb.h> + #include <drm/drm_gem.h> #include <drm/intel/display_parent_interface.h> @@ -8,6 +10,10 @@ #include "xe_bo.h" #include "xe_display_bo.h" #include "xe_pxp.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_wa.h" + +#include <generated/xe_device_wa_oob.h> static bool xe_display_bo_is_protected(struct drm_gem_object *obj) { @@ -42,7 +48,8 @@ static int xe_display_bo_framebuffer_init(struct drm_gem_object *obj, if (ret) goto err; - if (!(bo->flags & XE_BO_FLAG_FORCE_WC)) { + if (!(bo->flags & XE_BO_FLAG_FORCE_WC) && + bo->ttm.type != ttm_bo_type_sg) { /* * XE_BO_FLAG_FORCE_WC should ideally be set at creation, or is * automatically set when creating FB. We cannot change caching @@ -101,6 +108,114 @@ xe_display_bo_framebuffer_lookup(struct drm_device *drm, return gem; } +#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) +/* + * FIXME: There shouldn't be any reason to have XE_PAGE_SIZE stride + * alignment. The same 64 as i915 uses should be fine, and we shouldn't need to + * have driver specific values. However, dropping the stride alignment to 64 + * leads to underflowing the bo pin count in the atomic cleanup work. + */ +static u32 xe_display_bo_fbdev_pitch_align(u32 stride) +{ + return ALIGN(stride, XE_PAGE_SIZE); +} + +bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size) +{ + struct ttm_resource_manager *stolen; + + stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); + if (!stolen) + return false; + + if (IS_DGFX(xe)) + return false; + + if (XE_DEVICE_WA(xe, 22019338487_display)) + return false; + + /* + * If the FB is too big, just don't use it since fbdev is not very + * important and we should probably use that space with FBC or other + * features. + */ + return stolen->size >= (size * 2) >> PAGE_SHIFT; +} + +static struct drm_gem_object *xe_display_bo_fbdev_create(struct drm_device *drm, int size) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_bo *obj; + + obj = ERR_PTR(-ENODEV); + + if (xe_display_bo_fbdev_prefer_stolen(xe, size)) { + obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), + size, + ttm_bo_type_kernel, + XE_BO_FLAG_FORCE_WC | + XE_BO_FLAG_STOLEN | + XE_BO_FLAG_GGTT, + false); + if (!IS_ERR(obj)) + drm_info(&xe->drm, "Allocated fbdev into stolen\n"); + else + drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj)); + } else { + drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n"); + } + + if (IS_ERR(obj)) { + obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size, + ttm_bo_type_kernel, + XE_BO_FLAG_FORCE_WC | + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | + XE_BO_FLAG_GGTT, + false); + } + + if (IS_ERR(obj)) { + drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj); + return ERR_PTR(-ENOMEM); + } + + return &obj->ttm.base; +} + +static void xe_display_bo_fbdev_destroy(struct drm_gem_object *obj) +{ + xe_bo_unpin_map_no_vm(gem_to_xe_bo(obj)); +} + +static int xe_display_bo_fbdev_fill_info(struct drm_gem_object *_obj, struct fb_info *info, + struct i915_vma *vma) +{ + struct xe_bo *obj = gem_to_xe_bo(_obj); + struct pci_dev *pdev = to_pci_dev(_obj->dev->dev); + + if (!(obj->flags & XE_BO_FLAG_SYSTEM)) { + if (obj->flags & XE_BO_FLAG_STOLEN) + info->fix.smem_start = xe_ttm_stolen_io_offset(obj, 0); + else + info->fix.smem_start = + pci_resource_start(pdev, 2) + + xe_bo_addr(obj, 0, XE_PAGE_SIZE); + + info->fix.smem_len = obj->ttm.base.size; + } else { + /* XXX: Pure fiction, as the BO may not be physically accessible.. */ + info->fix.smem_start = 0; + info->fix.smem_len = obj->ttm.base.size; + } + XE_WARN_ON(iosys_map_is_null(&obj->vmap)); + + info->screen_base = obj->vmap.vaddr_iomem; + info->screen_size = obj->ttm.base.size; + + return 0; +} +#endif + const struct intel_display_bo_interface xe_display_bo_interface = { .is_protected = xe_display_bo_is_protected, .key_check = xe_pxp_obj_key_check, @@ -109,4 +224,10 @@ const struct intel_display_bo_interface xe_display_bo_interface = { .framebuffer_init = xe_display_bo_framebuffer_init, .framebuffer_fini = xe_display_bo_framebuffer_fini, .framebuffer_lookup = xe_display_bo_framebuffer_lookup, +#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) + .fbdev_create = xe_display_bo_fbdev_create, + .fbdev_destroy = xe_display_bo_fbdev_destroy, + .fbdev_fill_info = xe_display_bo_fbdev_fill_info, + .fbdev_pitch_align = xe_display_bo_fbdev_pitch_align, +#endif }; diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.h b/drivers/gpu/drm/xe/display/xe_display_bo.h index 6879c104b0b1..c72056884ff4 100644 --- a/drivers/gpu/drm/xe/display/xe_display_bo.h +++ b/drivers/gpu/drm/xe/display/xe_display_bo.h @@ -4,6 +4,12 @@ #ifndef __XE_DISPLAY_BO_H__ #define __XE_DISPLAY_BO_H__ +#include <linux/types.h> + +struct xe_device; + +bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size); + extern const struct intel_display_bo_interface xe_display_bo_interface; #endif diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index e45a1e7a4670..5f4a0cd8deca 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -3,17 +3,21 @@ * Copyright © 2021 Intel Corporation */ +#include <drm/intel/display_parent_interface.h> #include <drm/ttm/ttm_bo.h> -#include "intel_display_core.h" -#include "intel_display_types.h" +/* FIXME move the types to parent interface? */ +#include "i915_gtt_view_types.h" + +/* FIXME move intel_remapped_info_size() & co. to parent interface? */ #include "intel_fb.h" -#include "intel_fb_pin.h" -#include "intel_fbdev.h" + #include "xe_bo.h" #include "xe_device.h" #include "xe_display_vma.h" +#include "xe_fb_pin.h" #include "xe_ggtt.h" +#include "xe_pat.h" #include "xe_pm.h" #include "xe_vram_types.h" @@ -24,7 +28,7 @@ write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ struct xe_device *xe = xe_bo_device(bo); struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; u32 column, row; - u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); + u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, * by writing dpt/ggtt in a different order? @@ -64,7 +68,7 @@ write_dpt_remapped_linear(struct xe_bo *bo, struct iosys_map *map, struct xe_device *xe = xe_bo_device(bo); struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, - xe->pat.idx[XE_CACHE_NONE]); + xe_cache_pat_idx(xe, XE_CACHE_NONE)); unsigned int offset = plane->offset * XE_PAGE_SIZE; unsigned int size = plane->size; @@ -87,7 +91,7 @@ write_dpt_remapped_tiled(struct xe_bo *bo, struct iosys_map *map, struct xe_device *xe = xe_bo_device(bo); struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, - xe->pat.idx[XE_CACHE_NONE]); + xe_cache_pat_idx(xe, XE_CACHE_NONE)); unsigned int offset, column, row; for (row = 0; row < plane->height; row++) { @@ -139,22 +143,21 @@ write_dpt_remapped(struct xe_bo *bo, } } -static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, - const struct i915_gtt_view *view, - struct i915_vma *vma, - unsigned int alignment) +static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma *vma) { - struct xe_device *xe = to_xe_device(fb->base.dev); + struct xe_device *xe = to_xe_device(obj->dev); struct xe_tile *tile0 = xe_device_get_root_tile(xe); struct xe_ggtt *ggtt = tile0->mem.ggtt; - struct drm_gem_object *obj = intel_fb_bo(&fb->base); + const struct i915_gtt_view *view = pin_params->view; struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; u32 dpt_size, size = bo->ttm.base.size; if (view->type == I915_GTT_VIEW_NORMAL) dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); else if (view->type == I915_GTT_VIEW_REMAPPED) - dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, + dpt_size = ALIGN(intel_remapped_info_size(&view->remapped) * 8, XE_PAGE_SIZE); else /* display uses 4K tiles instead of bytes here, convert to entries.. */ @@ -168,7 +171,7 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, XE_BO_FLAG_VRAM0 | XE_BO_FLAG_GGTT | XE_BO_FLAG_PAGETABLE, - alignment, false); + pin_params->alignment, false); else dpt = xe_bo_create_pin_map_at_novm(xe, tile0, dpt_size, ~0ull, @@ -176,7 +179,7 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, XE_BO_FLAG_STOLEN | XE_BO_FLAG_GGTT | XE_BO_FLAG_PAGETABLE, - alignment, false); + pin_params->alignment, false); if (IS_ERR(dpt)) dpt = xe_bo_create_pin_map_at_novm(xe, tile0, dpt_size, ~0ull, @@ -185,12 +188,12 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, XE_BO_FLAG_GGTT | XE_BO_FLAG_PAGETABLE | XE_BO_FLAG_FORCE_WC, - alignment, false); + pin_params->alignment, false); if (IS_ERR(dpt)) return PTR_ERR(dpt); if (view->type == I915_GTT_VIEW_NORMAL) { - u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); + u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); u32 x; for (x = 0; x < size / XE_PAGE_SIZE; x++) { @@ -268,14 +271,13 @@ static void write_ggtt_rotated_node(struct xe_ggtt *ggtt, struct xe_ggtt_node *n rot_info->plane[i].dst_stride); } -static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, - const struct i915_gtt_view *view, - struct i915_vma *vma, - unsigned int alignment) +static int __xe_pin_fb_vma_ggtt(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma *vma) { - struct drm_gem_object *obj = intel_fb_bo(&fb->base); + const struct i915_gtt_view *view = pin_params->view; struct xe_bo *bo = gem_to_xe_bo(obj); - struct xe_device *xe = to_xe_device(fb->base.dev); + struct xe_device *xe = to_xe_device(obj->dev); struct xe_tile *tile0 = xe_device_get_root_tile(xe); struct xe_ggtt *ggtt = tile0->mem.ggtt; u64 pte, size; @@ -287,7 +289,7 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, */ guard(xe_pm_runtime_noresume)(xe); - align = XE_PAGE_SIZE; + align = max(XE_PAGE_SIZE, pin_params->alignment); if (xe_bo_is_vram(bo) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) align = max(align, SZ_64K); @@ -306,7 +308,7 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, /* display uses tiles instead of bytes here, so convert it back.. */ size = intel_rotation_info_size(&view->rotated) * XE_PAGE_SIZE; - pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); + pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); vma->node = xe_ggtt_insert_node_transform(ggtt, bo, pte, ALIGN(size, align), align, view->type == I915_GTT_VIEW_NORMAL ? @@ -318,25 +320,26 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, return ret; } -static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, - const struct i915_gtt_view *view, - unsigned int alignment) +static struct i915_vma *__xe_pin_fb_vma(struct drm_gem_object *obj, bool is_dpt, + const struct intel_fb_pin_params *pin_params) { - struct drm_device *dev = fb->base.dev; - struct xe_device *xe = to_xe_device(dev); + struct xe_device *xe = to_xe_device(obj->dev); struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); - struct drm_gem_object *obj = intel_fb_bo(&fb->base); struct xe_bo *bo = gem_to_xe_bo(obj); struct xe_validation_ctx ctx; struct drm_exec exec; int ret = 0; + /* We reject creating !SCANOUT fb's, so this is weird.. */ + drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC) && + bo->ttm.type != ttm_bo_type_sg); + if (!vma) return ERR_PTR(-ENODEV); refcount_set(&vma->ref, 1); if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && - intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && + pin_params->needs_cpu_lmem_access && !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram; @@ -376,10 +379,10 @@ static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, goto err; vma->bo = bo; - if (intel_fb_uses_dpt(&fb->base)) - ret = __xe_pin_fb_vma_dpt(fb, view, vma, alignment); + if (is_dpt) + ret = __xe_pin_fb_vma_dpt(obj, pin_params, vma); else - ret = __xe_pin_fb_vma_ggtt(fb, view, vma, alignment); + ret = __xe_pin_fb_vma_ggtt(obj, pin_params, vma); if (ret) goto err_unpin; @@ -412,97 +415,94 @@ static void __xe_unpin_fb_vma(struct i915_vma *vma) kfree(vma); } -struct i915_vma * -intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, - const struct i915_gtt_view *view, - unsigned int alignment, - unsigned int phys_alignment, - unsigned int vtd_guard, - bool uses_fence, - unsigned long *out_flags) +int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_ggtt_vma, + u32 *out_offset, + int *out_fence_id) { - *out_flags = 0; + struct i915_vma *ggtt_vma; - return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, alignment); + ggtt_vma = __xe_pin_fb_vma(obj, false, pin_params); + if (IS_ERR(ggtt_vma)) + return PTR_ERR(ggtt_vma); + + *out_ggtt_vma = ggtt_vma; + *out_offset = xe_ggtt_node_addr(ggtt_vma->node); + if (out_fence_id) + *out_fence_id = -1; + + return 0; } -void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) +static void xe_fb_pin_ggtt_unpin(struct i915_vma *ggtt_vma, + int fence_id) { - __xe_unpin_fb_vma(vma); + WARN_ON(fence_id >= 0); + + __xe_unpin_fb_vma(ggtt_vma); } -static bool reuse_vma(struct intel_plane_state *new_plane_state, - const struct intel_plane_state *old_plane_state) +static int xe_fb_pin_dpt_pin(struct drm_gem_object *obj, struct intel_dpt *dpt, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_dpt_vma, + struct i915_vma **out_ggtt_vma, + u32 *out_offset) { - struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb); - struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); - struct xe_device *xe = to_xe_device(fb->base.dev); - struct intel_display *display = xe->display; - struct i915_vma *vma; - - if (old_plane_state->hw.fb == new_plane_state->hw.fb && - !memcmp(&old_plane_state->view.gtt, - &new_plane_state->view.gtt, - sizeof(new_plane_state->view.gtt))) { - vma = old_plane_state->ggtt_vma; - goto found; - } - - if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) { - vma = intel_fbdev_vma_pointer(display->fbdev.fbdev); - if (vma) - goto found; - } + struct i915_vma *ggtt_vma; - return false; + WARN_ON(dpt); -found: - refcount_inc(&vma->ref); - new_plane_state->ggtt_vma = vma; + ggtt_vma = __xe_pin_fb_vma(obj, true, pin_params); + if (IS_ERR(ggtt_vma)) + return PTR_ERR(ggtt_vma); - new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) + - plane->surf_offset(new_plane_state); + *out_dpt_vma = NULL; /* not used on xe */ + *out_ggtt_vma = ggtt_vma; + *out_offset = xe_ggtt_node_addr(ggtt_vma->node); - return true; + return 0; } -int intel_plane_pin_fb(struct intel_plane_state *new_plane_state, - const struct intel_plane_state *old_plane_state) +static void xe_fb_pin_dpt_unpin(struct intel_dpt *dpt, + struct i915_vma *dpt_vma, + struct i915_vma *ggtt_vma) { - struct drm_framebuffer *fb = new_plane_state->hw.fb; - struct drm_gem_object *obj = intel_fb_bo(fb); - struct xe_bo *bo = gem_to_xe_bo(obj); - struct i915_vma *vma; - struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); - unsigned int alignment = plane->min_alignment(plane, fb, 0); + WARN_ON(dpt || dpt_vma); - if (reuse_vma(new_plane_state, old_plane_state)) - return 0; - - /* We reject creating !SCANOUT fb's, so this is weird.. */ - drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC)); - - vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment); - - if (IS_ERR(vma)) - return PTR_ERR(vma); + __xe_unpin_fb_vma(ggtt_vma); +} - new_plane_state->ggtt_vma = vma; +static struct i915_vma * +xe_fb_pin_reuse_vma(struct i915_vma *old_ggtt_vma, + struct drm_gem_object *old_obj, + const struct i915_gtt_view *old_view, + struct drm_gem_object *new_obj, + const struct i915_gtt_view *new_view, + u32 *out_offset) +{ + if (old_ggtt_vma && old_obj == new_obj && + !memcmp(old_view, new_view, sizeof(*new_view))) { + refcount_inc(&old_ggtt_vma->ref); - new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) + - plane->surf_offset(new_plane_state); + *out_offset = xe_ggtt_node_addr(old_ggtt_vma->node); - return 0; -} + return old_ggtt_vma; + } -void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) -{ - __xe_unpin_fb_vma(old_plane_state->ggtt_vma); - old_plane_state->ggtt_vma = NULL; + return NULL; } -void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map) +static void xe_fb_pin_get_map(struct i915_vma *vma, struct iosys_map *map) { *map = vma->bo->vmap; } + +const struct intel_display_fb_pin_interface xe_display_fb_pin_interface = { + .ggtt_pin = xe_fb_pin_ggtt_pin, + .ggtt_unpin = xe_fb_pin_ggtt_unpin, + .dpt_pin = xe_fb_pin_dpt_pin, + .dpt_unpin = xe_fb_pin_dpt_unpin, + .reuse_vma = xe_fb_pin_reuse_vma, + .get_map = xe_fb_pin_get_map, +}; diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.h b/drivers/gpu/drm/xe/display/xe_fb_pin.h new file mode 100644 index 000000000000..20dd8a99a25f --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2026 Intel Corporation */ + +#ifndef __XE_FB_PIN_H__ +#define __XE_FB_PIN_H__ + +#include <linux/types.h> + +struct drm_gem_object; +struct i915_vma; +struct intel_fb_pin_params; + +int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_ggtt_vma, + u32 *out_offset, + int *out_fence_id); + +extern const struct intel_display_fb_pin_interface xe_display_fb_pin_interface; + +#endif /* __XE_FB_PIN_H__ */ diff --git a/drivers/gpu/drm/xe/display/xe_initial_plane.c b/drivers/gpu/drm/xe/display/xe_initial_plane.c index 8bcae552dddc..02b46cc3b6df 100644 --- a/drivers/gpu/drm/xe/display/xe_initial_plane.c +++ b/drivers/gpu/drm/xe/display/xe_initial_plane.c @@ -7,34 +7,20 @@ #include "regs/xe_gtt_defs.h" -#include "intel_crtc.h" -#include "intel_display_regs.h" -#include "intel_display_types.h" +/* FIXME move intel_remapped_info_size() & co. */ #include "intel_fb.h" -#include "intel_fb_pin.h" -#include "intel_fbdev_fb.h" + +/* FIXME move intel_initial_plane_config */ +#include "intel_display_types.h" + #include "xe_bo.h" +#include "xe_display_bo.h" #include "xe_display_vma.h" +#include "xe_fb_pin.h" #include "xe_ggtt.h" #include "xe_mmio.h" #include "xe_vram_types.h" -/* Early xe has no irq */ -static void xe_initial_plane_vblank_wait(struct drm_crtc *_crtc) -{ - struct intel_crtc *crtc = to_intel_crtc(_crtc); - struct xe_device *xe = to_xe_device(crtc->base.dev); - struct xe_reg pipe_frmtmstmp = XE_REG(i915_mmio_reg_offset(PIPE_FRMTMSTMP(crtc->pipe))); - u32 timestamp; - int ret; - - timestamp = xe_mmio_read32(xe_root_tile_mmio(xe), pipe_frmtmstmp); - - ret = xe_mmio_wait32_not(xe_root_tile_mmio(xe), pipe_frmtmstmp, ~0U, timestamp, 40000U, ×tamp, false); - if (ret < 0) - drm_warn(&xe->drm, "waiting for early vblank failed with %i\n", ret); -} - static struct xe_bo * initial_plane_bo(struct xe_device *xe, struct intel_initial_plane_config *plane_config) @@ -86,7 +72,8 @@ initial_plane_bo(struct xe_device *xe, flags |= XE_BO_FLAG_STOLEN; if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) && - !intel_fbdev_fb_prefer_stolen(&xe->drm, plane_config->size)) { + IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) && + !xe_display_bo_fbdev_prefer_stolen(xe, plane_config->size)) { drm_info(&xe->drm, "Initial FB size exceeds half of stolen, discarding\n"); return NULL; } @@ -114,7 +101,7 @@ xe_alloc_initial_plane_obj(struct drm_device *drm, { struct xe_device *xe = to_xe_device(drm); struct drm_mode_fb_cmd2 mode_cmd = { 0 }; - struct drm_framebuffer *fb = &plane_config->fb->base; + struct drm_framebuffer *fb = plane_config->fb; struct xe_bo *bo; mode_cmd.pixel_format = fb->format->format; @@ -151,15 +138,19 @@ xe_initial_plane_setup(struct drm_plane_state *_plane_state, { struct intel_plane_state *plane_state = to_intel_plane_state(_plane_state); struct i915_vma *vma; + struct intel_fb_pin_params pin_params = { + .view = &plane_state->view.gtt, + }; + u32 offset; + int ret; - vma = intel_fb_pin_to_ggtt(fb, &plane_state->view.gtt, - 0, 0, 0, false, &plane_state->flags); - if (IS_ERR(vma)) - return PTR_ERR(vma); + ret = xe_fb_pin_ggtt_pin(intel_fb_bo(fb), &pin_params, &vma, &offset, NULL); + if (ret) + return ret; plane_state->ggtt_vma = vma; - plane_state->surf = xe_ggtt_node_addr(plane_state->ggtt_vma->node); + plane_state->surf = offset; plane_config->vma = vma; @@ -171,7 +162,6 @@ static void xe_plane_config_fini(struct intel_initial_plane_config *plane_config } const struct intel_display_initial_plane_interface xe_display_initial_plane_interface = { - .vblank_wait = xe_initial_plane_vblank_wait, .alloc_obj = xe_alloc_initial_plane_obj, .setup = xe_initial_plane_setup, .config_fini = xe_plane_config_fini, diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index 885fcf211e6d..18d0fde8c98f 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -20,7 +20,6 @@ #define XY_FAST_COLOR_BLT_CMD (2 << 29 | 0x44 << 22) #define XY_FAST_COLOR_BLT_DEPTH_32 (2 << 19) -#define XY_FAST_COLOR_BLT_DW 16 #define XY_FAST_COLOR_BLT_MOCS_MASK GENMASK(27, 22) #define XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK GENMASK(27, 24) #define XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT 31 @@ -31,14 +30,13 @@ #define XY_FAST_COPY_BLT_D1_DST_TILE4 REG_BIT(30) #define XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK GENMASK(23, 20) -#define MEM_COPY_CMD (2 << 29 | 0x5a << 22 | 0x8) +#define MEM_COPY_CMD (2 << 29 | 0x5a << 22) #define MEM_COPY_PAGE_COPY_MODE REG_BIT(19) #define MEM_COPY_MATRIX_COPY REG_BIT(17) #define MEM_COPY_SRC_MOCS_INDEX_MASK GENMASK(31, 28) #define MEM_COPY_DST_MOCS_INDEX_MASK GENMASK(6, 3) #define PVC_MEM_SET_CMD (2 << 29 | 0x5b << 22) -#define PVC_MEM_SET_CMD_LEN_DW 7 #define PVC_MEM_SET_MATRIX REG_BIT(17) #define PVC_MEM_SET_DATA_FIELD GENMASK(31, 24) /* Bspec lists field as [6:0], but index alone is from [6:1] */ diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h index 1b4a7e9a703d..94033982e694 100644 --- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -165,18 +165,21 @@ #define CTX_CTRL_INHIBIT_SYN_CTX_SWITCH REG_BIT(3) #define CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT REG_BIT(0) -#define RING_MODE(base) XE_REG((base) + 0x29c) -#define GFX_DISABLE_LEGACY_MODE REG_BIT(3) +#define GFX_MODE(base) XE_REG((base) + 0x29c, XE_REG_OPTION_MASKED) #define GFX_MSIX_INTERRUPT_ENABLE REG_BIT(13) +#define GFX_DISABLE_LEGACY_MODE REG_BIT(3) #define RING_CSMQDEBUG(base) XE_REG((base) + 0x2b0) +#define CURRENT_ACTIVE_QUEUE_ID_MASK REG_GENMASK(7, 0) + +#define RING_QUEUE_TIMESTAMP(base) XE_REG((base) + 0x4c0) +#define RING_QUEUE_TIMESTAMP_UDW(base) XE_REG((base) + 0x4c0 + 4) #define RING_TIMESTAMP(base) XE_REG((base) + 0x358) #define RING_TIMESTAMP_UDW(base) XE_REG((base) + 0x358 + 4) #define RING_VALID_MASK 0x00000001 #define RING_VALID 0x00000001 -#define STOP_RING REG_BIT(8) #define RING_CTX_TIMESTAMP(base) XE_REG((base) + 0x3a8) #define RING_CTX_TIMESTAMP_UDW(base) XE_REG((base) + 0x3ac) diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 353fe0bd49bf..08251c7a1a4b 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -530,6 +530,7 @@ #define ROW_CHICKEN3 XE_REG_MCR(0xe49c, XE_REG_OPTION_MASKED) #define XE2_EUPEND_CHK_FLUSH_DIS REG_BIT(14) +#define DIS_EU_GRF_POISON_TO_LSC REG_BIT(13) #define DIS_FIX_EOT1_FLUSH REG_BIT(9) #define TDL_TSL_CHICKEN XE_REG_MCR(0xe4c4, XE_REG_OPTION_MASKED) @@ -537,6 +538,9 @@ #define SLM_WMTP_RESTORE REG_BIT(11) #define RES_CHK_SPR_DIS REG_BIT(6) +#define TDL_TSL_CHICKEN2 XE_REG_MCR(0xe4cc, XE_REG_OPTION_MASKED) +#define TILEY_LOCALID REG_BIT(2) + #define ROW_CHICKEN XE_REG_MCR(0xe4f0, XE_REG_OPTION_MASKED) #define UGM_BACKUP_MODE REG_BIT(13) #define MDQ_ARBITRATION_MODE REG_BIT(12) @@ -562,6 +566,7 @@ #define DIS_ATOMIC_CHAINING_TYPED_WRITES REG_BIT(3) #define TDL_CHICKEN XE_REG_MCR(0xe5f4, XE_REG_OPTION_MASKED) +#define BIT_APQ_OPT_DIS REG_BIT(14) #define QID_WAIT_FOR_THREAD_NOT_RUN_DISABLE REG_BIT(12) #define EUSTALL_PERF_SAMPLING_DISABLE REG_BIT(5) diff --git a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h index 4d83461e538b..d6bc19ef277b 100644 --- a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h +++ b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h @@ -9,7 +9,11 @@ #define XELPG_GGTT_PTE_PAT0 BIT_ULL(52) #define XELPG_GGTT_PTE_PAT1 BIT_ULL(53) -#define XE_PTE_ADDR_MASK GENMASK_ULL(51, 12) +/* + * Mask for PTE address bits [51:shift]. + * shift is the lower address boundary of page. + */ +#define XE_PAGE_ADDR_MASK(shift) GENMASK_ULL(51, (shift)) #define GGTT_PTE_VFID GENMASK_ULL(11, 2) #define GUC_GGTT_TOP 0xFEE00000 diff --git a/drivers/gpu/drm/xe/regs/xe_irq_regs.h b/drivers/gpu/drm/xe/regs/xe_irq_regs.h index 9d74f454d3ff..1d6b976c4de0 100644 --- a/drivers/gpu/drm/xe/regs/xe_irq_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_irq_regs.h @@ -22,6 +22,7 @@ #define DISPLAY_IRQ REG_BIT(16) #define SOC_H2DMEMINT_IRQ REG_BIT(13) #define I2C_IRQ REG_BIT(12) +#define SYSCTRL_IRQ REG_BIT(11) #define GT_DW_IRQ(x) REG_BIT(x) /* diff --git a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h index b5eff383902c..4ab86fc369fd 100644 --- a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h +++ b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h @@ -34,6 +34,9 @@ #define CTX_CS_INT_VEC_REG 0x5a #define CTX_CS_INT_VEC_DATA (CTX_CS_INT_VEC_REG + 1) +#define CTX_QUEUE_TIMESTAMP (0xd0 + 1) +#define CTX_QUEUE_TIMESTAMP_UDW (0xd2 + 1) + #define INDIRECT_CTX_RING_HEAD (0x02 + 1) #define INDIRECT_CTX_RING_TAIL (0x04 + 1) #define INDIRECT_CTX_RING_START (0x06 + 1) diff --git a/drivers/gpu/drm/xe/regs/xe_oa_regs.h b/drivers/gpu/drm/xe/regs/xe_oa_regs.h index 04a729e610aa..aa66af7e99fe 100644 --- a/drivers/gpu/drm/xe/regs/xe_oa_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_oa_regs.h @@ -6,6 +6,9 @@ #ifndef __XE_OA_REGS__ #define __XE_OA_REGS__ +#define SYS_MEM_LAT_MEASURE XE_REG(0x145194) +#define SYS_MEM_LAT_MEASURE_EN REG_BIT(31) + #define RPM_CONFIG1 XE_REG(0xd04) #define GT_NOA_ENABLE REG_BIT(9) diff --git a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h index 4b3c46eb858f..c63b409d7a82 100644 --- a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h @@ -27,4 +27,7 @@ #define TEMP_SIGN_MASK REG_BIT(31) #define BMG_PACKAGE_TEMPERATURE XE_REG(0x138434) +#define CRI_PACKAGE_ENERGY_STATUS XE_REG(0x138120) +#define CRI_PLATFORM_ENERGY_STATUS XE_REG(0x138458) + #endif /* _XE_PCODE_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h b/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h new file mode 100644 index 000000000000..59f3f3ec59a6 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_REGS_H_ +#define _XE_SYSCTRL_REGS_H_ + +#include "xe_regs.h" + +#define SYSCTRL_BASE_OFFSET 0xdb000 +#define SYSCTRL_BASE (SOC_BASE + SYSCTRL_BASE_OFFSET) +#define SYSCTRL_MAILBOX_INDEX 0x03 +#define SYSCTRL_BAR_LENGTH 0x1000 + +#define SYSCTRL_MB_CTRL XE_REG(0x10) +#define SYSCTRL_MB_CTRL_RUN_BUSY REG_BIT(31) +#define SYSCTRL_MB_CTRL_IRQ REG_BIT(30) +#define SYSCTRL_MB_CTRL_RUN_BUSY_OUT REG_BIT(29) +#define SYSCTRL_MB_CTRL_PARAM3_MASK REG_GENMASK(28, 24) +#define SYSCTRL_MB_CTRL_PARAM2_MASK REG_GENMASK(23, 16) +#define SYSCTRL_MB_CTRL_PARAM1_MASK REG_GENMASK(15, 8) +#define SYSCTRL_MB_CTRL_COMMAND_MASK REG_GENMASK(7, 0) +#define SYSCTRL_MB_CTRL_CMD REG_FIELD_PREP(SYSCTRL_MB_CTRL_COMMAND_MASK, 5) + +#define SYSCTRL_MB_DATA0 XE_REG(0x14) +#define SYSCTRL_MB_DATA1 XE_REG(0x18) +#define SYSCTRL_MB_DATA2 XE_REG(0x1c) +#define SYSCTRL_MB_DATA3 XE_REG(0x20) + +#define SYSCTRL_FRAME_PHASE REG_BIT(24) +#define SYSCTRL_FRAME_CURRENT_MASK REG_GENMASK(21, 16) +#define SYSCTRL_FRAME_TOTAL_MASK REG_GENMASK(13, 8) +#define SYSCTRL_FRAME_COMMAND_MASK REG_GENMASK(7, 0) + +#endif diff --git a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c index efa8963ec248..e6eaa94d4d30 100644 --- a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c +++ b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c @@ -13,11 +13,28 @@ #define TEST_MAX_VFS 63 #define TEST_VRAM 0x7a800000ull /* random size that works on 32-bit */ +static bool xe_device_is_admin_only_stub_enable(const struct xe_device *xe) +{ + return true; +} + +static bool xe_device_is_admin_only_stub_disable(const struct xe_device *xe) +{ + return false; +} + static void pf_set_admin_mode(struct xe_device *xe, bool enable) { - /* should match logic of xe_sriov_pf_admin_only() */ - xe->sriov.pf.admin_only = enable; + typeof(xe_device_is_admin_only) *stub = enable ? + xe_device_is_admin_only_stub_enable : + xe_device_is_admin_only_stub_disable; + + kunit_activate_static_stub(kunit_get_current_test(), + xe_device_is_admin_only, + *stub); + KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_device_is_admin_only(xe)); } static void pf_set_usable_vram(struct xe_device *xe, u64 usable) diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 34e2f0f4631f..3c1be809be82 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -9,6 +9,7 @@ #include "tests/xe_kunit_helpers.h" #include "tests/xe_pci_test.h" +#include "xe_pat.h" #include "xe_pci.h" #include "xe_pm.h" @@ -246,7 +247,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test, /* First part of the test, are we updating our pagetable bo with a new entry? */ xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64, 0xdeaddeadbeefbeef); - expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe->pat.idx[XE_CACHE_WB], 0); + expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe_cache_pat_idx(xe, XE_CACHE_WB), 0); if (m->q->vm->flags & XE_VM_FLAG_64K) expected |= XE_PTE_PS64; if (xe_bo_is_vram(pt)) @@ -420,7 +421,7 @@ static struct dma_fence *blt_copy(struct xe_tile *tile, avail_pts, avail_pts); /* Add copy commands size here */ - batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + + batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) + ((xe_device_has_flat_ccs(xe) && copy_only_ccs) ? EMIT_COPY_CCS_DW : 0); bb = xe_bb_new(gt, batch_size, xe->info.has_usm); diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index f3179b31f13e..c2c686aed1cb 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -9,7 +9,6 @@ #include <kunit/test-bug.h> #include <kunit/test.h> -#include <kunit/test-bug.h> #include <kunit/visibility.h> #define PLATFORM_CASE(platform__, graphics_step__) \ @@ -131,12 +130,12 @@ static const char *subplatform_prefix(enum xe_subplatform s) return s == XE_SUBPLATFORM_NONE ? "" : " "; } -static const char *step_prefix(enum xe_step step) +static const char *step_prefix(enum intel_step step) { return step == STEP_NONE ? "" : " "; } -static const char *step_name(enum xe_step step) +static const char *step_name(enum intel_step step) { return step == STEP_NONE ? "" : xe_step_name(step); } @@ -311,6 +310,11 @@ const void *xe_pci_id_gen_param(struct kunit *test, const void *prev, char *desc } EXPORT_SYMBOL_IF_KUNIT(xe_pci_id_gen_param); +static void fake_init_devid(struct xe_device *xe) +{ + /* Nothing to do, just keep zero. */ +} + static int fake_read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, u32 *ver, u32 *revid) { @@ -369,6 +373,7 @@ done: xe->sriov.__mode = data && data->sriov_mode ? data->sriov_mode : XE_SRIOV_MODE_NONE; + kunit_activate_static_stub(test, init_devid, fake_init_devid); kunit_activate_static_stub(test, read_gmdid, fake_read_gmdid); kunit_activate_static_stub(test, xe_info_probe_tile_count, fake_xe_info_probe_tile_count); diff --git a/drivers/gpu/drm/xe/tests/xe_rtp.c b/drivers/gpu/drm/xe/tests/xe_rtp.c new file mode 100644 index 000000000000..b3a8b75936d1 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_rtp.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include "tests/xe_rtp_test.h" + +#include <kunit/visibility.h> + +/** + * xe_rtp_rule_matches - Check if a set of RTP rule set match against the + * device/GT/hwe + * @xe: The xe device + * @gt: The GT struct (may be NULL) + * @hwe: The hw_engine (may be NULL) + * @rules: The array of rules to match against + * @n_rules: Number of items in @rules + * @err: Pointer (may be NULL) to set error number. + * + * This parses the set of rules and check if they match against the passed + * parameters. + * + * If passed, @err is updated with a non-zero negative error number or zero if + * no errors were found during the parsing/evaluation of rules. + * + * Returns true if there is a match and false if there is no match or if an + * error was found. + */ +bool xe_rtp_rule_matches(const struct xe_device *xe, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_rule *rules, + unsigned int n_rules, + int *err) +{ + return rule_matches_with_err(xe, gt, hwe, rules, n_rules, err); +} +EXPORT_SYMBOL_IF_KUNIT(xe_rtp_rule_matches); diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_test.c b/drivers/gpu/drm/xe/tests/xe_rtp_test.c index e5a0f985a700..3d0688d058d9 100644 --- a/drivers/gpu/drm/xe/tests/xe_rtp_test.c +++ b/drivers/gpu/drm/xe/tests/xe_rtp_test.c @@ -9,28 +9,43 @@ #include <drm/drm_drv.h> #include <drm/drm_kunit_helpers.h> +#include <kunit/static_stub.h> #include <kunit/test.h> #include "regs/xe_gt_regs.h" #include "regs/xe_reg_defs.h" #include "xe_device.h" #include "xe_device_types.h" +#include "xe_gt_mcr.h" #include "xe_kunit_helpers.h" #include "xe_pci_test.h" #include "xe_reg_sr.h" #include "xe_rtp.h" - -#define REGULAR_REG1 XE_REG(1) -#define REGULAR_REG2 XE_REG(2) -#define REGULAR_REG3 XE_REG(3) -#define MCR_REG1 XE_REG_MCR(1) -#define MCR_REG2 XE_REG_MCR(2) -#define MCR_REG3 XE_REG_MCR(3) -#define MASKED_REG1 XE_REG(1, XE_REG_OPTION_MASKED) +#include "xe_rtp_test.h" + +#define REGULAR_REG1 XE_REG(1) +#define REGULAR_REG2 XE_REG(2) +#define REGULAR_REG3 XE_REG(3) +#define REGULAR_REG4 XE_REG(4) +#define BAD_REGULAR_REG5 XE_REG(5) +#define MCR_REG1 XE_REG_MCR(1) +#define MCR_REG2 XE_REG_MCR(2) +#define MCR_REG3 XE_REG_MCR(3) +#define BAD_MCR_REG4 XE_REG_MCR(4) +#define MCR_REG5 XE_REG_MCR(5) +#define MASKED_REG1 XE_REG(1, XE_REG_OPTION_MASKED) #undef XE_REG_MCR #define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1) +struct rtp_rules_test_case { + const char *name; + bool expected_match; + int expected_err; + const struct xe_rtp_rule *rules; + u8 n_rules; +}; + struct rtp_to_sr_test_case { const char *name; struct xe_reg expected_reg; @@ -39,15 +54,32 @@ struct rtp_to_sr_test_case { unsigned long expected_count_sr_entries; unsigned int expected_sr_errors; unsigned long expected_active; - const struct xe_rtp_entry_sr *entries; + const struct xe_rtp_table_sr table; }; struct rtp_test_case { const char *name; unsigned long expected_active; - const struct xe_rtp_entry *entries; + const struct xe_rtp_table table; }; +static bool fake_xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg) +{ + /* + * All supported platforms in this imaginary setup will always have REG4 + * as a non-MCR register and REG5 as MCR, meaning that BAD_MCR_REG4 and + * BAD_REGULAR_REG5 represent programming errors to be captured by our + * tests. + */ + if (reg.raw == BAD_REGULAR_REG5.raw) + return true; + + if (reg.raw == BAD_MCR_REG4.raw) + return false; + + return reg.mcr; +} + static bool match_yes(const struct xe_device *xe, const struct xe_gt *gt, const struct xe_hw_engine *hwe) { @@ -60,6 +92,194 @@ static bool match_no(const struct xe_device *xe, const struct xe_gt *gt, return false; } +static const struct rtp_rules_test_case rtp_rules_cases[] = { + /* + * Single rules. + * + * TODO: Include other types of rules as well: GRAPHICS_VERSION(), + * MEDIA_VERSION(), etc. + */ + { + .name = "no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no)), + }, + { + .name = "yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes)), + }, + + /* Conjunctions with 2 operands. */ + { + .name = "no-and-no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no), FUNC(match_no)), + }, + { + .name = "no-and-yes", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no), FUNC(match_yes)), + }, + { + .name = "yes-and-no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_yes), FUNC(match_no)), + }, + { + .name = "yes-and-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), FUNC(match_yes)), + }, + + /* Disjunctions with 2 operands. */ + { + .name = "no-or-no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no), OR, FUNC(match_no)), + }, + { + .name = "no-or-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_no), OR, FUNC(match_yes)), + }, + { + .name = "yes-or-no", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), OR, FUNC(match_no)), + }, + { + .name = "yes-or-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), OR, FUNC(match_yes)), + }, + + /* Conjunction and disjunctions. */ + { + .name = "no-yes-or-yes-no", + .expected_match = false, + XE_RTP_RULES(FUNC(match_no), FUNC(match_yes), OR, + FUNC(match_yes), FUNC(match_no)), + }, + { + .name = "no-yes-or-yes-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_no), FUNC(match_yes), OR, + FUNC(match_yes), FUNC(match_yes)), + }, + { + .name = "yes-yes-or-no-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), FUNC(match_yes), OR, + FUNC(match_no), FUNC(match_yes)), + }, + { + .name = "yes-yes-or-yes-yes", + .expected_match = true, + XE_RTP_RULES(FUNC(match_yes), FUNC(match_yes), OR, + FUNC(match_yes), FUNC(match_yes)), + }, + { + .name = "no-no-or-yes-or-no", + .expected_match = true, + XE_RTP_RULES(FUNC(match_no), FUNC(match_no), OR, + FUNC(match_yes), OR, + FUNC(match_no)), + }, + + /* Syntax errors. */ + { + .name = "or", + .expected_match = false, + .expected_err = -EINVAL, + XE_RTP_RULES(OR), + }, + { + .name = "or-yes", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(OR, FUNC(match_yes)), + }, + { + .name = "or-no", + .expected_match = false, + .expected_err = -EINVAL, + XE_RTP_RULES(OR, FUNC(match_no)), + }, + { + .name = "yes-or", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_yes), OR), + }, + { + .name = "no-or", + .expected_match = false, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_no), OR), + }, + { + .name = "no-or-or-yes", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_no), OR, OR, FUNC(match_yes)), + }, + { + .name = "yes-or-or-no", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_yes), OR, OR, FUNC(match_no)), + }, + { + .name = "no-or-or-no", + .expected_match = false, + .expected_err = -EINVAL, + XE_RTP_RULES(FUNC(match_no), OR, OR, FUNC(match_no)), + }, + + /* No match because hwe is NULL. */ + { + .name = "missing-context-engine-class", + .expected_match = false, + XE_RTP_RULES(ENGINE_CLASS(RENDER)), + }, + + /* + * Missing context (hwe==NULL) does not cause parsing to stop, hence we + * expect a match. + */ + { + .name = "missing-context-engine-class-or-yes", + .expected_match = true, + XE_RTP_RULES(ENGINE_CLASS(RENDER), OR, FUNC(match_yes)), + }, + + /* + * Missing context (hwe==NULL) does not cause parsing to stop, hence we + * expect a syntax error. + */ + { + .name = "missing-context-engine-class-or-or-yes", + .expected_match = true, + .expected_err = -EINVAL, + XE_RTP_RULES(ENGINE_CLASS(RENDER), OR, OR, FUNC(match_yes)), + }, +}; + +static void xe_rtp_rules_tests(struct kunit *test) +{ + const struct rtp_rules_test_case *param = test->param_value; + struct xe_device *xe = test->priv; + struct xe_gt *gt = xe_device_get_root_tile(xe)->primary_gt; + int err; + bool match; + + match = xe_rtp_rule_matches(xe, gt, NULL, param->rules, param->n_rules, &err); + + KUNIT_EXPECT_EQ(test, match, param->expected_match); + KUNIT_EXPECT_EQ(test, err, param->expected_err); +} + static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { { .name = "coalesce-same-reg", @@ -69,7 +289,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { .expected_active = BIT(0) | BIT(1), .expected_count_sr_entries = 1, /* Different bits on the same register: create a single entry */ - .entries = (const struct xe_rtp_entry_sr[]) { + .table = XE_RTP_TABLE_SR( { XE_RTP_NAME("basic-1"), XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) @@ -78,8 +298,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(1))) }, - {} - }, + ), }, { .name = "no-match-no-add", @@ -89,7 +308,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { .expected_active = BIT(0), .expected_count_sr_entries = 1, /* Don't coalesce second entry since rules don't match */ - .entries = (const struct xe_rtp_entry_sr[]) { + .table = XE_RTP_TABLE_SR( { XE_RTP_NAME("basic-1"), XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) @@ -98,82 +317,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { XE_RTP_RULES(FUNC(match_no)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(1))) }, - {} - }, - }, - { - .name = "match-or", - .expected_reg = REGULAR_REG1, - .expected_set_bits = REG_BIT(0) | REG_BIT(1) | REG_BIT(2), - .expected_clr_bits = REG_BIT(0) | REG_BIT(1) | REG_BIT(2), - .expected_active = BIT(0) | BIT(1) | BIT(2), - .expected_count_sr_entries = 1, - .entries = (const struct xe_rtp_entry_sr[]) { - { XE_RTP_NAME("first"), - XE_RTP_RULES(FUNC(match_yes), OR, FUNC(match_no)), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) - }, - { XE_RTP_NAME("middle"), - XE_RTP_RULES(FUNC(match_no), FUNC(match_no), OR, - FUNC(match_yes), OR, - FUNC(match_no)), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(1))) - }, - { XE_RTP_NAME("last"), - XE_RTP_RULES(FUNC(match_no), OR, FUNC(match_yes)), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(2))) - }, - { XE_RTP_NAME("no-match"), - XE_RTP_RULES(FUNC(match_no), OR, FUNC(match_no)), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(3))) - }, - {} - }, - }, - { - .name = "match-or-xfail", - .expected_reg = REGULAR_REG1, - .expected_count_sr_entries = 0, - .entries = (const struct xe_rtp_entry_sr[]) { - { XE_RTP_NAME("leading-or"), - XE_RTP_RULES(OR, FUNC(match_yes)), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) - }, - { XE_RTP_NAME("trailing-or"), - /* - * First condition is match_no, otherwise the failure - * wouldn't really trigger as RTP stops processing as - * soon as it has a matching set of rules - */ - XE_RTP_RULES(FUNC(match_no), OR), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(1))) - }, - { XE_RTP_NAME("no-or-or-yes"), - XE_RTP_RULES(FUNC(match_no), OR, OR, FUNC(match_yes)), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(2))) - }, - {} - }, - }, - { - .name = "no-match-no-add-multiple-rules", - .expected_reg = REGULAR_REG1, - .expected_set_bits = REG_BIT(0), - .expected_clr_bits = REG_BIT(0), - .expected_active = BIT(0), - .expected_count_sr_entries = 1, - /* Don't coalesce second entry due to one of the rules */ - .entries = (const struct xe_rtp_entry_sr[]) { - { XE_RTP_NAME("basic-1"), - XE_RTP_RULES(FUNC(match_yes)), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) - }, - { XE_RTP_NAME("basic-2"), - XE_RTP_RULES(FUNC(match_yes), FUNC(match_no)), - XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(1))) - }, - {} - }, + ), }, { .name = "two-regs-two-entries", @@ -183,7 +327,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { .expected_active = BIT(0) | BIT(1), .expected_count_sr_entries = 2, /* Same bits on different registers are not coalesced */ - .entries = (const struct xe_rtp_entry_sr[]) { + .table = XE_RTP_TABLE_SR( { XE_RTP_NAME("basic-1"), XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) @@ -192,8 +336,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG2, REG_BIT(0))) }, - {} - }, + ), }, { .name = "clr-one-set-other", @@ -203,7 +346,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { .expected_active = BIT(0) | BIT(1), .expected_count_sr_entries = 1, /* Check clr vs set actions on different bits */ - .entries = (const struct xe_rtp_entry_sr[]) { + .table = XE_RTP_TABLE_SR( { XE_RTP_NAME("basic-1"), XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) @@ -212,8 +355,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(CLR(REGULAR_REG1, REG_BIT(1))) }, - {} - }, + ), }, { #define TEMP_MASK REG_GENMASK(10, 8) @@ -225,14 +367,13 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { .expected_active = BIT(0), .expected_count_sr_entries = 1, /* Check FIELD_SET works */ - .entries = (const struct xe_rtp_entry_sr[]) { + .table = XE_RTP_TABLE_SR( { XE_RTP_NAME("basic-1"), XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(FIELD_SET(REGULAR_REG1, TEMP_MASK, TEMP_FIELD)) }, - {} - }, + ), #undef TEMP_MASK #undef TEMP_FIELD }, @@ -244,7 +385,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { .expected_active = BIT(0) | BIT(1), .expected_count_sr_entries = 1, .expected_sr_errors = 1, - .entries = (const struct xe_rtp_entry_sr[]) { + .table = XE_RTP_TABLE_SR( { XE_RTP_NAME("basic-1"), XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) @@ -254,8 +395,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) }, - {} - }, + ), }, { .name = "conflict-not-disjoint", @@ -265,7 +405,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { .expected_active = BIT(0) | BIT(1), .expected_count_sr_entries = 1, .expected_sr_errors = 1, - .entries = (const struct xe_rtp_entry_sr[]) { + .table = XE_RTP_TABLE_SR( { XE_RTP_NAME("basic-1"), XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) @@ -275,8 +415,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(CLR(REGULAR_REG1, REG_GENMASK(1, 0))) }, - {} - }, + ), }, { .name = "conflict-reg-type", @@ -286,7 +425,7 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { .expected_active = BIT(0) | BIT(1) | BIT(2), .expected_count_sr_entries = 1, .expected_sr_errors = 2, - .entries = (const struct xe_rtp_entry_sr[]) { + .table = XE_RTP_TABLE_SR( { XE_RTP_NAME("basic-1"), XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(REGULAR_REG1, REG_BIT(0))) @@ -301,8 +440,37 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { XE_RTP_RULES(FUNC(match_yes)), XE_RTP_ACTIONS(SET(MASKED_REG1, REG_BIT(0))) }, - {} - }, + ), + }, + { + .name = "bad-mcr-reg-forced-to-regular", + .expected_reg = REGULAR_REG4, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("bad-mcr-regular-reg"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(BAD_MCR_REG4, REG_BIT(0))) + }, + ), + }, + { + .name = "bad-regular-reg-forced-to-mcr", + .expected_reg = MCR_REG5, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .table = XE_RTP_TABLE_SR( + { XE_RTP_NAME("bad-regular-reg"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(BAD_REGULAR_REG5, REG_BIT(0))) + }, + ), }, }; @@ -314,16 +482,12 @@ static void xe_rtp_process_to_sr_tests(struct kunit *test) struct xe_reg_sr *reg_sr = >->reg_sr; const struct xe_reg_sr_entry *sre, *sr_entry = NULL; struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); - unsigned long idx, count_sr_entries = 0, count_rtp_entries = 0, active = 0; + unsigned long idx, count_sr_entries = 0, active = 0; xe_reg_sr_init(reg_sr, "xe_rtp_to_sr_tests", xe); - while (param->entries[count_rtp_entries].rules) - count_rtp_entries++; - - xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, count_rtp_entries); - xe_rtp_process_to_sr(&ctx, param->entries, count_rtp_entries, - reg_sr, false); + xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, param->table.n_entries); + xe_rtp_process_to_sr(&ctx, ¶m->table, reg_sr, false); xa_for_each(®_sr->xa, idx, sre) { if (idx == param->expected_reg.addr) @@ -356,119 +520,77 @@ static const struct rtp_test_case rtp_cases[] = { { .name = "active1", .expected_active = BIT(0), - .entries = (const struct xe_rtp_entry[]) { + .table = XE_RTP_TABLE( { XE_RTP_NAME("r1"), XE_RTP_RULES(FUNC(match_yes)), }, - {} - }, + ), }, { .name = "active2", .expected_active = BIT(0) | BIT(1), - .entries = (const struct xe_rtp_entry[]) { + .table = XE_RTP_TABLE( { XE_RTP_NAME("r1"), XE_RTP_RULES(FUNC(match_yes)), }, { XE_RTP_NAME("r2"), XE_RTP_RULES(FUNC(match_yes)), }, - {} - }, + ), }, { .name = "active-inactive", .expected_active = BIT(0), - .entries = (const struct xe_rtp_entry[]) { + .table = XE_RTP_TABLE( { XE_RTP_NAME("r1"), XE_RTP_RULES(FUNC(match_yes)), }, { XE_RTP_NAME("r2"), XE_RTP_RULES(FUNC(match_no)), }, - {} - }, + ), }, { .name = "inactive-active", .expected_active = BIT(1), - .entries = (const struct xe_rtp_entry[]) { + .table = XE_RTP_TABLE( { XE_RTP_NAME("r1"), XE_RTP_RULES(FUNC(match_no)), }, { XE_RTP_NAME("r2"), XE_RTP_RULES(FUNC(match_yes)), }, - {} - }, + ), }, { - .name = "inactive-1st_or_active-inactive", + .name = "inactive-active-inactive", .expected_active = BIT(1), - .entries = (const struct xe_rtp_entry[]) { + .table = XE_RTP_TABLE( { XE_RTP_NAME("r1"), XE_RTP_RULES(FUNC(match_no)), }, - { XE_RTP_NAME("r2_or_conditions"), - XE_RTP_RULES(FUNC(match_yes), OR, - FUNC(match_no), OR, - FUNC(match_no)) }, - { XE_RTP_NAME("r3"), - XE_RTP_RULES(FUNC(match_no)), - }, - {} - }, - }, - { - .name = "inactive-2nd_or_active-inactive", - .expected_active = BIT(1), - .entries = (const struct xe_rtp_entry[]) { - { XE_RTP_NAME("r1"), - XE_RTP_RULES(FUNC(match_no)), + { XE_RTP_NAME("r2"), + XE_RTP_RULES(FUNC(match_yes)), }, - { XE_RTP_NAME("r2_or_conditions"), - XE_RTP_RULES(FUNC(match_no), OR, - FUNC(match_yes), OR, - FUNC(match_no)) }, { XE_RTP_NAME("r3"), XE_RTP_RULES(FUNC(match_no)), }, - {} - }, + ), }, { - .name = "inactive-last_or_active-inactive", - .expected_active = BIT(1), - .entries = (const struct xe_rtp_entry[]) { + .name = "inactive-inactive-inactive", + .expected_active = 0, + .table = XE_RTP_TABLE( { XE_RTP_NAME("r1"), XE_RTP_RULES(FUNC(match_no)), }, - { XE_RTP_NAME("r2_or_conditions"), - XE_RTP_RULES(FUNC(match_no), OR, - FUNC(match_no), OR, - FUNC(match_yes)) }, - { XE_RTP_NAME("r3"), - XE_RTP_RULES(FUNC(match_no)), - }, - {} - }, - }, - { - .name = "inactive-no_or_active-inactive", - .expected_active = 0, - .entries = (const struct xe_rtp_entry[]) { - { XE_RTP_NAME("r1"), + { XE_RTP_NAME("r2"), XE_RTP_RULES(FUNC(match_no)), }, - { XE_RTP_NAME("r2_or_conditions"), - XE_RTP_RULES(FUNC(match_no), OR, - FUNC(match_no), OR, - FUNC(match_no)) }, { XE_RTP_NAME("r3"), XE_RTP_RULES(FUNC(match_no)), }, - {} - }, + ), }, }; @@ -478,17 +600,21 @@ static void xe_rtp_process_tests(struct kunit *test) struct xe_device *xe = test->priv; struct xe_gt *gt = xe_device_get_root_tile(xe)->primary_gt; struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); - unsigned long count_rtp_entries = 0, active = 0; - - while (param->entries[count_rtp_entries].rules) - count_rtp_entries++; + unsigned long active = 0; - xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, count_rtp_entries); - xe_rtp_process(&ctx, param->entries); + xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, param->table.n_entries); + xe_rtp_process(&ctx, ¶m->table); KUNIT_EXPECT_EQ(test, active, param->expected_active); } +static void rtp_rules_desc(const struct rtp_rules_test_case *t, char *desc) +{ + strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(rtp_rules, rtp_rules_cases, rtp_rules_desc); + static void rtp_to_sr_desc(const struct rtp_to_sr_test_case *t, char *desc) { strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); @@ -523,6 +649,8 @@ static int xe_rtp_test_init(struct kunit *test) xe->drm.dev = dev; test->priv = xe; + kunit_activate_static_stub(test, xe_gt_mcr_check_reg, fake_xe_gt_mcr_check_reg); + return 0; } @@ -534,6 +662,7 @@ static void xe_rtp_test_exit(struct kunit *test) } static struct kunit_case xe_rtp_tests[] = { + KUNIT_CASE_PARAM(xe_rtp_rules_tests, rtp_rules_gen_params), KUNIT_CASE_PARAM(xe_rtp_process_to_sr_tests, rtp_to_sr_gen_params), KUNIT_CASE_PARAM(xe_rtp_process_tests, rtp_gen_params), {} diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_test.h b/drivers/gpu/drm/xe/tests/xe_rtp_test.h new file mode 100644 index 000000000000..26becc1e2af0 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_rtp_test.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_RTP_TEST_H_ +#define _XE_RTP_TEST_H_ + +#include <linux/types.h> + +struct xe_device; +struct xe_gt; +struct xe_hw_engine; +struct xe_rtp_rule; + +bool xe_rtp_rule_matches(const struct xe_device *xe, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_rule *rules, + unsigned int n_rules, + int *err); + +#endif diff --git a/drivers/gpu/drm/xe/tests/xe_wa_test.c b/drivers/gpu/drm/xe/tests/xe_wa_test.c index 49d191043dfa..ff0e2502b39f 100644 --- a/drivers/gpu/drm/xe/tests/xe_wa_test.c +++ b/drivers/gpu/drm/xe/tests/xe_wa_test.c @@ -9,6 +9,8 @@ #include <kunit/test.h> #include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" #include "xe_kunit_helpers.h" #include "xe_pci_test.h" #include "xe_reg_sr.h" @@ -19,8 +21,10 @@ static int xe_wa_test_init(struct kunit *test) { const struct xe_pci_fake_data *param = test->param_value; struct xe_pci_fake_data data = *param; - struct xe_device *xe; struct device *dev; + struct xe_device *xe; + struct xe_gt *gt; + int id; int ret; dev = drm_kunit_helper_alloc_device(test); @@ -33,6 +37,12 @@ static int xe_wa_test_init(struct kunit *test) ret = xe_pci_fake_device_init(xe); KUNIT_ASSERT_EQ(test, ret, 0); + /* Needed for sanitize_mcr(). */ + for_each_gt(gt, xe, id) { + xe_gt_mcr_init_early(gt); + xe_gt_mmio_init(gt); + } + if (!param->graphics_verx100) xe->info.step = param->step; @@ -55,7 +65,7 @@ static void xe_wa_gt(struct kunit *test) xe_wa_process_gt(gt); xe_tuning_process_gt(gt); - KUNIT_ASSERT_EQ(test, gt->reg_sr.errors, 0); + KUNIT_EXPECT_EQ(test, gt->reg_sr.errors, 0); } } diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 6b518858538f..7ed76349075f 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -173,19 +173,6 @@ mem_type_to_migrate(struct xe_device *xe, u32 mem_type) return tile->migrate; } -static struct xe_vram_region *res_to_mem_region(struct ttm_resource *res) -{ - struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); - struct ttm_resource_manager *mgr; - struct xe_ttm_vram_mgr *vram_mgr; - - xe_assert(xe, resource_is_vram(res)); - mgr = ttm_manager_type(&xe->ttm, res->mem_type); - vram_mgr = to_xe_ttm_vram_mgr(mgr); - - return container_of(vram_mgr, struct xe_vram_region, ttm); -} - static void try_add_system(struct xe_device *xe, struct xe_bo *bo, u32 bo_flags, u32 *c) { @@ -599,11 +586,17 @@ static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt) kfree(tt); } -static bool xe_ttm_resource_visible(struct ttm_resource *mem) +static bool xe_ttm_resource_visible(struct xe_device *xe, struct ttm_resource *mem) { - struct xe_ttm_vram_mgr_resource *vres = - to_xe_ttm_vram_mgr_resource(mem); + struct xe_ttm_vram_mgr_resource *vres; + + if (mem->mem_type == XE_PL_STOLEN) { + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; + + return mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe); + } + vres = to_xe_ttm_vram_mgr_resource(mem); return vres->used_visible_size == mem->size; } @@ -621,7 +614,7 @@ bool xe_bo_is_visible_vram(struct xe_bo *bo) if (drm_WARN_ON(bo->ttm.base.dev, !xe_bo_is_vram(bo))) return false; - return xe_ttm_resource_visible(bo->ttm.resource); + return xe_ttm_resource_visible(xe_bo_device(bo), bo->ttm.resource); } static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, @@ -635,9 +628,9 @@ static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, return 0; case XE_PL_VRAM0: case XE_PL_VRAM1: { - struct xe_vram_region *vram = res_to_mem_region(mem); + struct xe_vram_region *vram = xe_map_resource_to_region(mem); - if (!xe_ttm_resource_visible(mem)) + if (!xe_ttm_resource_visible(xe, mem)) return -EINVAL; mem->bus.offset = mem->start << PAGE_SHIFT; @@ -1109,6 +1102,21 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, xe_pm_runtime_get_noresume(xe); } + /* + * Attach CCS BBs before submitting the copy job below so a VF + * migration racing the copy sees valid, up to date attach state. + */ + if (IS_VF_CCS_READY(xe) && + ((move_lacks_source && new_mem->mem_type == XE_PL_TT) || + (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) && + handle_system_ccs) { + ret = xe_sriov_vf_ccs_attach_bo(bo, new_mem); + if (ret) { + xe_pm_runtime_put(xe); + goto out; + } + } + if (move_lacks_source) { u32 flags = 0; @@ -1146,22 +1154,19 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, ttm_bo_move_null(ttm_bo, new_mem); } - dma_fence_put(fence); - xe_pm_runtime_put(xe); - /* - * CCS meta data is migrated from TT -> SMEM. So, let us detach the - * BBs from BO as it is no longer needed. + * Detach must wait for the copy above to complete: a VF migration + * racing an in-flight copy must still see valid CCS BBs, so don't + * tear them down until the copy fence has signaled. */ if (IS_VF_CCS_READY(xe) && old_mem_type == XE_PL_TT && - new_mem->mem_type == XE_PL_SYSTEM) + new_mem->mem_type == XE_PL_SYSTEM) { + dma_fence_wait(fence, false); xe_sriov_vf_ccs_detach_bo(bo); + } - if (IS_VF_CCS_READY(xe) && - ((move_lacks_source && new_mem->mem_type == XE_PL_TT) || - (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) && - handle_system_ccs) - ret = xe_sriov_vf_ccs_attach_bo(bo); + dma_fence_put(fence); + xe_pm_runtime_put(xe); out: if ((!ttm_bo->resource || ttm_bo->resource->mem_type == XE_PL_SYSTEM) && @@ -1356,7 +1361,7 @@ int xe_bo_notifier_prepare_pinned(struct xe_bo *bo) backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL, xe_bo_size(bo), DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS | - XE_BO_FLAG_PINNED, &exec); + XE_BO_FLAG_PINNED, NULL, &exec); if (IS_ERR(backup)) { drm_exec_retry_on_contention(&exec); ret = PTR_ERR(backup); @@ -1497,7 +1502,7 @@ int xe_bo_evict_pinned(struct xe_bo *bo) xe_bo_size(bo), DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS | - XE_BO_FLAG_PINNED, &exec); + XE_BO_FLAG_PINNED, NULL, &exec); if (IS_ERR(backup)) { drm_exec_retry_on_contention(&exec); ret = PTR_ERR(backup); @@ -1642,7 +1647,7 @@ static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo, if (ttm_bo->resource->mem_type == XE_PL_STOLEN) return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT; - vram = res_to_mem_region(ttm_bo->resource); + vram = xe_map_resource_to_region(ttm_bo->resource); xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor); return (vram->io_start + cursor.start) >> PAGE_SHIFT; } @@ -1782,7 +1787,7 @@ static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo, goto out; } - vram = res_to_mem_region(ttm_bo->resource); + vram = xe_map_resource_to_region(ttm_bo->resource); xe_res_first(ttm_bo->resource, offset & PAGE_MASK, xe_bo_size(bo) - (offset & PAGE_MASK), &cursor); @@ -1833,6 +1838,8 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo) if (bo->ttm.base.import_attach) drm_prime_gem_destroy(&bo->ttm.base, NULL); + if (bo->dma_buf) + dma_buf_put(bo->dma_buf); drm_gem_object_release(&bo->ttm.base); xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list)); @@ -2290,6 +2297,8 @@ void xe_bo_free(struct xe_bo *bo) * @cpu_caching: The cpu caching used for system memory backing store. * @type: The TTM buffer object type. * @flags: XE_BO_FLAG_ flags. + * @dma_buf: The dma-buf to reference for the BO lifetime (imported BOs), + * or NULL. * @exec: The drm_exec transaction to use for exhaustive eviction. * * Initialize or create an xe buffer object. On failure, any allocated buffer @@ -2301,7 +2310,8 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, struct xe_tile *tile, struct dma_resv *resv, struct ttm_lru_bulk_move *bulk, size_t size, u16 cpu_caching, enum ttm_bo_type type, - u32 flags, struct drm_exec *exec) + u32 flags, struct dma_buf *dma_buf, + struct drm_exec *exec) { struct ttm_operation_ctx ctx = { .interruptible = true, @@ -2390,6 +2400,17 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, placement = (type == ttm_bo_type_sg || bo->flags & XE_BO_FLAG_DEFER_BACKING) ? &sys_placement : &bo->placement; + + /* + * For imported BOs, keep the exporter dma-buf alive for the BO + * lifetime. Taken before ttm_bo_init_reserved() to also cover a + * creation failure there. Released in xe_ttm_bo_destroy(). + */ + if (dma_buf) { + get_dma_buf(dma_buf); + bo->dma_buf = dma_buf; + } + err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type, placement, alignment, &ctx, NULL, resv, xe_ttm_bo_destroy); @@ -2507,7 +2528,7 @@ __xe_bo_create_locked(struct xe_device *xe, vm && !xe_vm_in_fault_mode(vm) && flags & XE_BO_FLAG_USER ? &vm->lru_bulk_move : NULL, size, - cpu_caching, type, flags, exec); + cpu_caching, type, flags, NULL, exec); if (IS_ERR(bo)) return bo; @@ -2927,7 +2948,7 @@ uint64_t vram_region_gpu_offset(struct ttm_resource *res) case XE_PL_SYSTEM: return 0; default: - return res_to_mem_region(res)->dpa_base; + return xe_map_resource_to_region(res)->dpa_base; } return 0; } diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 6340317f7d2e..7ae1d9ac0574 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -118,7 +118,8 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, struct xe_tile *tile, struct dma_resv *resv, struct ttm_lru_bulk_move *bulk, size_t size, u16 cpu_caching, enum ttm_bo_type type, - u32 flags, struct drm_exec *exec); + u32 flags, struct dma_buf *dma_buf, + struct drm_exec *exec); struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags, diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index 077e35b4cdce..e45f24301050 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -36,6 +36,8 @@ struct xe_bo { struct xe_bo *backup_obj; /** @parent_obj: Ref to parent bo if this a backup_obj */ struct xe_bo *parent_obj; + /** @dma_buf: Imported dma-buf ref to keep its resv alive. */ + struct dma_buf *dma_buf; /** @flags: flags for this buffer object */ u32 flags; /** @vm: VM this BO is attached to, for extobj this will be NULL */ @@ -67,7 +69,7 @@ struct xe_bo { /** @attr: User controlled attributes for bo */ struct { /** - * @atomic_access: type of atomic access bo needs + * @attr.atomic_access: type of atomic access bo needs * protected by bo dma-resv lock */ u32 atomic_access; diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index ad2d8f179eb6..22b471303984 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -15,9 +15,11 @@ #include "xe_bo.h" #include "xe_device.h" #include "xe_force_wake.h" +#include "xe_gt.h" #include "xe_gt_debugfs.h" #include "xe_gt_printk.h" #include "xe_guc_ads.h" +#include "xe_hw_engine.h" #include "xe_mmio.h" #include "xe_pm.h" #include "xe_psmi.h" @@ -61,6 +63,37 @@ static struct xe_device *node_to_xe(struct drm_info_node *node) return to_xe_device(node->minor->dev); } +static void print_engine_class_mask(struct drm_printer *p, u16 mask) +{ + if (!mask) { + drm_printf(p, " none\n"); + return; + } + + for (enum xe_engine_class ec = 0; ec < XE_ENGINE_CLASS_MAX; ec++) { + if (mask & BIT(ec)) + drm_printf(p, " %s", xe_hw_engine_class_to_str(ec)); + } + drm_printf(p, "\n"); +} + +static void print_engine_mask(struct drm_printer *p, struct xe_gt *gt, u64 mask) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + if (!mask) { + drm_printf(p, " none\n"); + return; + } + + for_each_hw_engine(hwe, gt, id) { + if (mask & BIT_ULL(id)) + drm_printf(p, " %s", hwe->name); + } + drm_printf(p, "\n"); +} + static int info(struct seq_file *m, void *data) { struct xe_device *xe = node_to_xe(m->private); @@ -88,13 +121,15 @@ static int info(struct seq_file *m, void *data) drm_printf(&p, "has_flat_ccs %s\n", str_yes_no(xe->info.has_flat_ccs)); drm_printf(&p, "has_usm %s\n", str_yes_no(xe->info.has_usm)); drm_printf(&p, "skip_guc_pc %s\n", str_yes_no(xe->info.skip_guc_pc)); + drm_printf(&p, "multi_lrc_engine_classes"); + print_engine_class_mask(&p, xe->info.multi_lrc_mask); for_each_gt(gt, xe, id) { drm_printf(&p, "gt%d force wake %d\n", id, xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT)); - drm_printf(&p, "gt%d engine_mask 0x%llx\n", id, - gt->info.engine_mask); - drm_printf(&p, "gt%d multi_queue_engine_class_mask 0x%x\n", id, - gt->info.multi_queue_engine_class_mask); + drm_printf(&p, "gt%d engines", id); + print_engine_mask(&p, gt, gt->info.engine_mask); + drm_printf(&p, "gt%d multi_queue_engine_classes", id); + print_engine_class_mask(&p, gt->info.multi_queue_engine_class_mask); } return 0; diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.c b/drivers/gpu/drm/xe/xe_dep_scheduler.c index 51d99fee9aa5..004aac8b89e6 100644 --- a/drivers/gpu/drm/xe/xe_dep_scheduler.c +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.c @@ -78,7 +78,6 @@ xe_dep_scheduler_create(struct xe_device *xe, const struct drm_sched_init_args args = { .ops = &sched_ops, .submit_wq = submit_wq, - .num_rqs = 1, .credit_limit = job_limit, .timeout = MAX_SCHEDULE_TIMEOUT, .name = name, diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.h b/drivers/gpu/drm/xe/xe_dep_scheduler.h index 853961eec64b..f314fb5d80f5 100644 --- a/drivers/gpu/drm/xe/xe_dep_scheduler.h +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.h @@ -3,6 +3,9 @@ * Copyright © 2025 Intel Corporation */ +#ifndef _XE_DEP_SCHEDULER_H_ +#define _XE_DEP_SCHEDULER_H_ + #include <linux/types.h> struct drm_sched_entity; @@ -19,3 +22,5 @@ void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler); struct drm_sched_entity * xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler); + +#endif diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c index 558a1a9841a0..5f2b90b18f97 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.c +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -8,7 +8,7 @@ #include <linux/ascii85.h> #include <linux/devcoredump.h> -#include <generated/utsrelease.h> +#include <linux/utsname.h> #include <drm/drm_managed.h> @@ -101,7 +101,7 @@ static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count, drm_puts(&p, "**** Xe Device Coredump ****\n"); drm_printf(&p, "Reason: %s\n", ss->reason); - drm_puts(&p, "kernel: " UTS_RELEASE "\n"); + drm_printf(&p, "kernel: %s\n", init_utsname()->release); drm_puts(&p, "module: " KBUILD_MODNAME "\n"); ts = ktime_to_timespec64(ss->snapshot_time); @@ -149,7 +149,8 @@ static void xe_devcoredump_snapshot_free(struct xe_devcoredump_snapshot *ss) xe_guc_ct_snapshot_free(ss->guc.ct); ss->guc.ct = NULL; - xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc); + if (!IS_ERR_OR_NULL(ss->gt)) + xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc); ss->matched_node = NULL; xe_guc_exec_queue_snapshot_free(ss->ge); @@ -254,7 +255,8 @@ static void xe_devcoredump_free(void *data) if (!data || !coredump_to_xe(coredump)) return; - cancel_work_sync(&coredump->snapshot.work); + if (coredump->captured) + cancel_work_sync(&coredump->snapshot.work); mutex_lock(&coredump->lock); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index ffea4a453c01..dcb48caa485d 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -10,13 +10,13 @@ #include <linux/fault-inject.h> #include <linux/units.h> -#include <drm/drm_atomic_helper.h> #include <drm/drm_client.h> #include <drm/drm_gem_ttm_helper.h> #include <drm/drm_ioctl.h> #include <drm/drm_managed.h> #include <drm/drm_pagemap_util.h> #include <drm/drm_print.h> +#include <kunit/static_stub.h> #include <uapi/drm/xe_drm.h> #include "display/xe_display.h" @@ -25,6 +25,7 @@ #include "regs/xe_regs.h" #include "xe_bo.h" #include "xe_bo_evict.h" +#include "xe_configfs.h" #include "xe_debugfs.h" #include "xe_defaults.h" #include "xe_devcoredump.h" @@ -65,6 +66,7 @@ #include "xe_survivability_mode.h" #include "xe_sriov.h" #include "xe_svm.h" +#include "xe_sysctrl.h" #include "xe_tile.h" #include "xe_ttm_stolen_mgr.h" #include "xe_ttm_sys_mgr.h" @@ -389,8 +391,9 @@ bool xe_is_xe_file(const struct file *file) return file->f_op == &xe_driver_fops; } -static struct drm_driver driver = { +static const struct drm_driver regular_driver = { .driver_features = + XE_DISPLAY_DRIVER_FEATURES | DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA, @@ -412,8 +415,43 @@ static struct drm_driver driver = { .major = DRIVER_MAJOR, .minor = DRIVER_MINOR, .patchlevel = DRIVER_PATCHLEVEL, + XE_DISPLAY_DRIVER_OPS, }; +#ifdef CONFIG_PCI_IOV +static const struct drm_ioctl_desc xe_ioctls_admin_only[] = { + DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), +}; + +static const struct drm_driver admin_only_driver = { + .driver_features = + DRIVER_GEM | DRIVER_RENDER | DRIVER_GEM_GPUVA, + .open = xe_file_open, + .postclose = xe_file_close, + .ioctls = xe_ioctls_admin_only, + .num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only), + .fops = &xe_driver_fops, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, +}; + +/** + * xe_device_is_admin_only() - Check whether device is admin only or not. + * @xe: the &xe_device to check + * + * Return: true if the device is admin only, false otherwise. + */ +bool xe_device_is_admin_only(const struct xe_device *xe) +{ + KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe); + return xe->drm.driver == &admin_only_driver; +} +#endif + static void xe_device_destroy(struct drm_device *dev, void *dummy) { struct xe_device *xe = to_xe_device(dev); @@ -435,46 +473,77 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy) ttm_device_fini(&xe->ttm); } -struct xe_device *xe_device_create(struct pci_dev *pdev, - const struct pci_device_id *ent) +/** + * xe_device_create() - Create a new &xe_device instance + * @pdev: the parent &pci_dev + * + * Allocate and initialize a device managed Xe device structure. + * + * Return: pointer to new &xe_device on success, or ERR_PTR on failure. + */ +struct xe_device *xe_device_create(struct pci_dev *pdev) { + const struct drm_driver *driver = ®ular_driver; struct xe_device *xe; int err; - xe_display_driver_set_hooks(&driver); +#ifdef CONFIG_PCI_IOV + /* + * Since XE device is not initialized yet, read from configfs + * directly to decide whether we are in admin-only PF mode or not. + */ + if (xe_configfs_admin_only_pf(pdev)) + driver = &admin_only_driver; +#endif - err = aperture_remove_conflicting_pci_devices(pdev, driver.name); + err = aperture_remove_conflicting_pci_devices(pdev, driver->name); if (err) return ERR_PTR(err); - xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm); + xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm); if (IS_ERR(xe)) return xe; + err = xe_device_init_early(xe); + if (err) + return ERR_PTR(err); + + return xe; +} +ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ + +/** + * xe_device_init_early() - Initialize a new &xe_device instance + * @xe: the &xe_device to initialize + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_device_init_early(struct xe_device *xe) +{ + int err; + err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, xe->drm.anon_inode->i_mapping, - xe->drm.vma_offset_manager, 0); - if (WARN_ON(err)) - return ERR_PTR(err); + xe->drm.vma_offset_manager, + TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M))); + if (err) + return err; xe_bo_dev_init(&xe->bo_device); err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); if (err) - return ERR_PTR(err); + return err; err = xe_shrinker_create(xe); if (err) - return ERR_PTR(err); + return err; - xe->info.devid = pdev->device; - xe->info.revid = pdev->revision; - xe->info.force_execlist = xe_modparam.force_execlist; xe->atomic_svm_timeslice_ms = 5; xe->min_run_period_lr_ms = 5; err = xe_irq_init(xe); if (err) - return ERR_PTR(err); + return err; xe_validation_device_init(&xe->val); @@ -484,7 +553,7 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, err = xe_pagemap_shrinker_create(xe); if (err) - return ERR_PTR(err); + return err; xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); @@ -503,13 +572,13 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, err = xe_bo_pinned_init(xe); if (err) - return ERR_PTR(err); + return err; xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", WQ_MEM_RECLAIM); xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0); - xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU, 0); + xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU | WQ_MEM_RECLAIM, 0); if (!xe->ordered_wq || !xe->unordered_wq || !xe->preempt_fence_wq || !xe->destroy_wq) { /* @@ -517,16 +586,19 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, * drmm_add_action_or_reset register above */ drm_err(&xe->drm, "Failed to allocate xe workqueues\n"); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } err = drmm_mutex_init(&xe->drm, &xe->pmt.lock); if (err) - return ERR_PTR(err); + return err; - return xe; + err = xe_pm_init_early(xe); + if (err) + return err; + + return 0; } -ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ static bool xe_driver_flr_disabled(struct xe_device *xe) { @@ -707,6 +779,11 @@ int xe_device_probe_early(struct xe_device *xe) xe_sriov_probe_early(xe); + if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) { + xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n"); + return -ENODEV; + } + if (IS_SRIOV_VF(xe)) vf_update_device_info(xe); @@ -992,6 +1069,10 @@ int xe_device_probe(struct xe_device *xe) if (err) goto err_unregister_display; + err = xe_sysctrl_init(xe); + if (err) + goto err_unregister_display; + err = xe_device_sysfs_init(xe); if (err) goto err_unregister_display; diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index e4b9de8d8e95..975768a6a9c8 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -43,8 +43,8 @@ static inline struct xe_device *ttm_to_xe_device(struct ttm_device *ttm) return container_of(ttm, struct xe_device, ttm); } -struct xe_device *xe_device_create(struct pci_dev *pdev, - const struct pci_device_id *ent); +struct xe_device *xe_device_create(struct pci_dev *pdev); +int xe_device_init_early(struct xe_device *xe); int xe_device_probe_early(struct xe_device *xe); int xe_device_probe(struct xe_device *xe); void xe_device_remove(struct xe_device *xe); @@ -146,37 +146,37 @@ static inline struct xe_force_wake *gt_to_fw(struct xe_gt *gt) void xe_device_assert_mem_access(struct xe_device *xe); -static inline bool xe_device_has_flat_ccs(struct xe_device *xe) +static inline bool xe_device_has_flat_ccs(const struct xe_device *xe) { return xe->info.has_flat_ccs; } -static inline bool xe_device_has_sriov(struct xe_device *xe) +static inline bool xe_device_has_sriov(const struct xe_device *xe) { return xe->info.has_sriov; } -static inline bool xe_device_has_msix(struct xe_device *xe) +static inline bool xe_device_has_msix(const struct xe_device *xe) { return xe->irq.msix.nvec > 0; } -static inline bool xe_device_has_memirq(struct xe_device *xe) +static inline bool xe_device_has_memirq(const struct xe_device *xe) { return GRAPHICS_VERx100(xe) >= 1250; } -static inline bool xe_device_uses_memirq(struct xe_device *xe) +static inline bool xe_device_uses_memirq(const struct xe_device *xe) { return xe_device_has_memirq(xe) && (IS_SRIOV_VF(xe) || xe_device_has_msix(xe)); } -static inline bool xe_device_has_lmtt(struct xe_device *xe) +static inline bool xe_device_has_lmtt(const struct xe_device *xe) { return IS_DGFX(xe); } -static inline bool xe_device_has_mert(struct xe_device *xe) +static inline bool xe_device_has_mert(const struct xe_device *xe) { return xe->info.has_mert; } @@ -211,6 +211,15 @@ bool xe_is_xe_file(const struct file *file); struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid); +#ifdef CONFIG_PCI_IOV +bool xe_device_is_admin_only(const struct xe_device *xe); +#else +static inline bool xe_device_is_admin_only(const struct xe_device *xe) +{ + return false; +} +#endif + /* * Occasionally it is seen that the G2H worker starts running after a delay of more than * a second even after being queued and activated by the Linux workqueue subsystem. This diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 615218d775b1..aad10899d9ac 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -27,6 +27,7 @@ #include "xe_sriov_vf_ccs_types.h" #include "xe_step_types.h" #include "xe_survivability_mode_types.h" +#include "xe_sysctrl_types.h" #include "xe_tile_types.h" #include "xe_validation.h" @@ -41,6 +42,7 @@ struct xe_ggtt; struct xe_i2c; struct xe_pat_ops; struct xe_pxp; +struct xe_ttm_stolen_mgr; struct xe_vram_region; /** @@ -196,6 +198,8 @@ struct xe_device { u8 has_soc_remapper_telem:1; /** @info.has_sriov: Supports SR-IOV */ u8 has_sriov:1; + /** @info.has_sysctrl: Supports System Controller */ + u8 has_sysctrl:1; /** @info.has_usm: Device has unified shared memory support */ u8 has_usm:1; /** @info.has_64bit_timestamp: Device supports 64-bit timestamps */ @@ -215,8 +219,6 @@ struct xe_device { u8 probe_display:1; /** @info.skip_guc_pc: Skip GuC based PM feature init */ u8 skip_guc_pc:1; - /** @info.skip_mtcfg: skip Multi-Tile configuration from MTCFG register */ - u8 skip_mtcfg:1; /** @info.skip_pcode: skip access to PCODE uC */ u8 skip_pcode:1; /** @info.needs_shared_vf_gt_wq: needs shared GT WQ on VF */ @@ -273,8 +275,10 @@ struct xe_device { struct xe_vram_region *vram; /** @mem.sys_mgr: system TTM manager */ struct ttm_resource_manager sys_mgr; - /** @mem.sys_mgr: system memory shrinker. */ + /** @mem.shrinker: system memory shrinker. */ struct xe_shrinker *shrinker; + /** @mem.stolen_mgr: stolen memory manager. */ + struct xe_ttm_stolen_mgr *stolen_mgr; } mem; /** @sriov: device level virtualization data */ @@ -295,7 +299,7 @@ struct xe_device { /** @usm: unified memory state */ struct { - /** @usm.asid: convert a ASID to VM */ + /** @usm.asid_to_vm: convert an ASID to VM */ struct xarray asid_to_vm; /** @usm.next_asid: next ASID, used to cyclical alloc asids */ u32 next_asid; @@ -312,7 +316,7 @@ struct xe_device { /** @usm.pf_queue: Page fault queues */ struct xe_pagefault_queue pf_queue[XE_PAGEFAULT_QUEUE_COUNT]; #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) - /** @usm.pagemap_shrinker: Shrinker for unused pagemaps */ + /** @usm.dpagemap_shrinker: Shrinker for unused pagemaps */ struct drm_pagemap_shrinker *dpagemap_shrinker; #endif } usm; @@ -334,7 +338,7 @@ struct xe_device { struct list_head kernel_bo_present; /** @pinned.late.evicted: pinned BO that have been evicted */ struct list_head evicted; - /** @pinned.external: pinned external and dma-buf. */ + /** @pinned.late.external: pinned external and dma-buf. */ struct list_head external; } late; } pinned; @@ -351,7 +355,7 @@ struct xe_device { /** @unordered_wq: used to serialize unordered work */ struct workqueue_struct *unordered_wq; - /** @destroy_wq: used to serialize user destroy work, like queue */ + /** @destroy_wq: used to serialize SVM pagemap destroy work */ struct workqueue_struct *destroy_wq; /** @tiles: device tiles */ @@ -369,7 +373,7 @@ struct xe_device { struct { /** * @mem_access.vram_userfault.lock: Protects access to - * @vram_usefault.list Using mutex instead of spinlock + * @mem_access.vram_userfault.list Using mutex instead of spinlock * as lock is applied to entire list operation which * may sleep */ @@ -400,7 +404,7 @@ struct xe_device { const struct xe_pat_table_entry *pat_primary_pta; /** @pat.pat_media_pta: media GT PAT entry for page table accesses */ const struct xe_pat_table_entry *pat_media_pta; - u32 idx[__XE_CACHE_LEVEL_COUNT]; + u16 idx[__XE_CACHE_LEVEL_COUNT]; } pat; /** @d3cold: Encapsulate d3cold related stuff */ @@ -508,6 +512,9 @@ struct xe_device { /** @i2c: I2C host controller */ struct xe_i2c *i2c; + /** @sc: System Controller */ + struct xe_sysctrl sc; + /** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */ u32 atomic_svm_timeslice_ms; @@ -578,7 +585,7 @@ struct xe_file { /** @vm: VM state for file */ struct { - /** @vm.xe: xarray to store VMs */ + /** @vm.xa: xarray to store VMs */ struct xarray xa; /** * @vm.lock: Protects VM lookup + reference and removal from diff --git a/drivers/gpu/drm/xe/xe_device_wa_oob.rules b/drivers/gpu/drm/xe/xe_device_wa_oob.rules index d129cddb6ead..d8dc41851425 100644 --- a/drivers/gpu/drm/xe/xe_device_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_device_wa_oob.rules @@ -4,3 +4,6 @@ 22019338487_display PLATFORM(LUNARLAKE) 14022085890 SUBPLATFORM(BATTLEMAGE, G21) 14026539277 PLATFORM(NOVALAKE_P), PLATFORM_STEP(A0, B0) +14026633728 PLATFORM(CRESCENTISLAND) +14026746987 PLATFORM(CRESCENTISLAND) +14026779378 PLATFORM(CRESCENTISLAND) diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index 8a920e58245c..bf0728838ead 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -302,7 +302,7 @@ xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size, 0, /* Will require 1way or 2way for vm_bind */ - ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec); + ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, dma_buf, &exec); drm_exec_retry_on_contention(&exec); if (IS_ERR(bo)) { ret = PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c index 84b66147bf49..81020b4b344e 100644 --- a/drivers/gpu/drm/xe/xe_drm_client.c +++ b/drivers/gpu/drm/xe/xe_drm_client.c @@ -168,10 +168,20 @@ static void bo_meminfo(struct xe_bo *bo, struct drm_memory_stats stats[TTM_NUM_MEM_TYPES]) { u64 sz = xe_bo_size(bo); - u32 mem_type = bo->ttm.resource->mem_type; + u32 mem_type; xe_bo_assert_held(bo); + /* + * The resource can be NULL if the BO has been purged, plus maybe some + * other cases. Either way there shouldn't be any memory to account for, + * or a current resource to account this against, so skip for now. + */ + if (!bo->ttm.resource) + return; + + mem_type = bo->ttm.resource->mem_type; + if (drm_gem_object_is_shared_for_memory_stats(&bo->ttm.base)) stats[mem_type].shared += sz; else diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c index c6cd32b7eeda..cd236f53699e 100644 --- a/drivers/gpu/drm/xe/xe_drm_ras.c +++ b/drivers/gpu/drm/xe/xe_drm_ras.c @@ -27,6 +27,16 @@ static int hw_query_error_counter(struct xe_drm_ras_counter *info, return 0; } +static int hw_clear_error_counter(struct xe_drm_ras_counter *info, u32 error_id) +{ + if (!info || !info[error_id].name) + return -ENOENT; + + atomic_set(&info[error_id].counter, 0); + + return 0; +} + static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_id, const char **name, u32 *val) { @@ -37,6 +47,15 @@ static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_ return hw_query_error_counter(info, error_id, name, val); } +static int clear_uncorrectable_error_counter(struct drm_ras_node *node, u32 error_id) +{ + struct xe_device *xe = node->priv; + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_UNCORRECTABLE]; + + return hw_clear_error_counter(info, error_id); +} + static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id, const char **name, u32 *val) { @@ -47,6 +66,15 @@ static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id return hw_query_error_counter(info, error_id, name, val); } +static int clear_correctable_error_counter(struct drm_ras_node *node, u32 error_id) +{ + struct xe_device *xe = node->priv; + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE]; + + return hw_clear_error_counter(info, error_id); +} + static struct xe_drm_ras_counter *allocate_and_copy_counters(struct xe_device *xe) { struct xe_drm_ras_counter *counter; @@ -92,10 +120,13 @@ static int assign_node_params(struct xe_device *xe, struct drm_ras_node *node, if (IS_ERR(ras->info[severity])) return PTR_ERR(ras->info[severity]); - if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) + if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) { node->query_error_counter = query_correctable_error_counter; - else + node->clear_error_counter = clear_correctable_error_counter; + } else { node->query_error_counter = query_uncorrectable_error_counter; + node->clear_error_counter = clear_uncorrectable_error_counter; + } return 0; } diff --git a/drivers/gpu/drm/xe/xe_drm_ras.h b/drivers/gpu/drm/xe/xe_drm_ras.h index 5cc8f0124411..365c70e93e82 100644 --- a/drivers/gpu/drm/xe/xe_drm_ras.h +++ b/drivers/gpu/drm/xe/xe_drm_ras.h @@ -2,8 +2,8 @@ /* * Copyright © 2026 Intel Corporation */ -#ifndef XE_DRM_RAS_H_ -#define XE_DRM_RAS_H_ +#ifndef _XE_DRM_RAS_H_ +#define _XE_DRM_RAS_H_ struct xe_device; diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index dddcdd0bb7a3..297be3c42b20 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -44,6 +44,7 @@ struct per_xecore_buf { struct xe_eu_stall_data_stream { bool pollin; bool enabled; + bool reset_detected; int wait_num_reports; int sampling_rate_mult; wait_queue_head_t poll_wq; @@ -428,9 +429,20 @@ static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream) set_bit(xecore, stream->data_drop.mask); xecore_buf->write = write_ptr; } + /* If a GT or engine reset happens during EU stall sampling, + * all EU stall registers get reset to 0 and the cached values of + * the EU stall data buffers' read pointers are out of sync with + * the register values. This causes invalid data to be returned + * from read(). To prevent this, check the value of a EU stall base + * register. If it is zero, there has been a reset. + */ + if (unlikely(!xe_gt_mcr_unicast_read_any(gt, XEHPC_EUSTALL_BASE))) + stream->reset_detected = true; + + stream->pollin = min_data_present || stream->reset_detected; mutex_unlock(&stream->xecore_buf_lock); - return min_data_present; + return stream->pollin; } static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 instance) @@ -544,6 +556,15 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st int ret = 0; mutex_lock(&stream->xecore_buf_lock); + /* If EU stall registers got reset due to a GT/engine reset, + * continuing with the read() will return invalid data to + * the user space. Just return -ENODEV instead. + */ + if (unlikely(stream->reset_detected)) { + xe_gt_dbg(gt, "EU stall base register has been reset\n"); + mutex_unlock(&stream->xecore_buf_lock); + return -ENODEV; + } if (bitmap_weight(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS)) { if (!stream->data_drop.reported_to_user) { stream->data_drop.reported_to_user = true; @@ -554,7 +575,6 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st } stream->data_drop.reported_to_user = false; } - for_each_dss_steering(xecore, gt, group, instance) { ret = xe_eu_stall_data_buf_read(stream, buf, count, &total_size, gt, group, instance, xecore); @@ -609,7 +629,8 @@ static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, * We don't want to block the next read() when there is data in the buffer * now, but couldn't be accommodated in the small user buffer. */ - stream->pollin = false; + if (!stream->reset_detected) + stream->pollin = false; return ret; } @@ -692,6 +713,7 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) xecore_buf->write = write_ptr; xecore_buf->read = write_ptr; } + stream->reset_detected = false; stream->data_drop.reported_to_user = false; bitmap_zero(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS); @@ -717,13 +739,13 @@ static void eu_stall_data_buf_poll_work_fn(struct work_struct *work) container_of(work, typeof(*stream), buf_poll_work.work); struct xe_gt *gt = stream->gt; - if (eu_stall_data_buf_poll(stream)) { - stream->pollin = true; + if (eu_stall_data_buf_poll(stream)) wake_up(&stream->poll_wq); - } - queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, - &stream->buf_poll_work, - msecs_to_jiffies(POLL_PERIOD_MS)); + + if (!stream->reset_detected) + queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, + &stream->buf_poll_work, + msecs_to_jiffies(POLL_PERIOD_MS)); } static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream, diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h index d1c76e503799..842bef9f6872 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.h +++ b/drivers/gpu/drm/xe/xe_eu_stall.h @@ -3,8 +3,8 @@ * Copyright © 2025 Intel Corporation */ -#ifndef __XE_EU_STALL_H__ -#define __XE_EU_STALL_H__ +#ifndef _XE_EU_STALL_H_ +#define _XE_EU_STALL_H_ #include "xe_gt_types.h" #include "xe_sriov.h" diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index e05dabfcd43c..d5293bc33a67 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -292,13 +292,23 @@ retry: goto err_exec; } - /* Wait behind rebinds */ + /* + * Wait behind rebinds and any kernel operations (evictions, defrag + * moves, ...) on the VM and all external BOs. The VM's private BOs + * carry their kernel ops in the VM dma-resv KERNEL slot, while each + * external BO carries them in its own dma-resv KERNEL slot; both are + * covered by iterating every object locked by the exec, mirroring the + * drm_gpuvm_resv_add_fence() below. + */ if (!xe_vm_in_lr_mode(vm)) { - err = xe_sched_job_add_deps(job, - xe_vm_resv(vm), - DMA_RESV_USAGE_KERNEL); - if (err) - goto err_put_job; + struct drm_gem_object *obj; + + drm_exec_for_each_locked_object(exec, obj) { + err = xe_sched_job_add_deps(job, obj->resv, + DMA_RESV_USAGE_KERNEL); + if (err) + goto err_put_job; + } } for (i = 0; i < num_syncs && !err; i++) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 071b8c41df43..1b5ca3ce578a 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -275,8 +275,12 @@ static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u { xe_assert(gt_to_xe(q->gt), idx < q->width); - scoped_guard(spinlock, &q->lrc_lookup_lock) + scoped_guard(spinlock, &q->lrc_lookup_lock) { q->lrc[idx] = lrc; + if (xe_exec_queue_is_multi_queue(q)) + q->lrc[idx]->multi_queue.primary_lrc = + q->multi_queue.group->primary->lrc[0]; + } } /** @@ -852,11 +856,6 @@ static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue * return 0; } -static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q) -{ - return q->gt->info.multi_queue_engine_class_mask & BIT(q->class); -} - static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q, u32 primary_id) { @@ -912,6 +911,7 @@ static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q } q->multi_queue.pos = pos; + q->lrc[0]->multi_queue.pos = pos; return 0; } @@ -931,7 +931,7 @@ static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queu static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q, u64 value) { - if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q))) + if (XE_IOCTL_DBG(xe, !xe_gt_supports_multi_queue(q->gt, q->class))) return -ENODEV; if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe))) diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 8ce78e0b1d50..2f5ccf294675 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -214,7 +214,7 @@ struct xe_exec_queue { */ struct xe_dep_scheduler *dep_scheduler; /** - * @last_fence: last fence for tlb invalidation, protected by + * @tlb_inval.last_fence: last fence for tlb invalidation, protected by * vm->lock in write mode */ struct dma_fence *last_fence; diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c index 755a2bff5d7b..9fb99c038ea8 100644 --- a/drivers/gpu/drm/xe/xe_execlist.c +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -47,7 +47,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, struct xe_mmio *mmio = >->mmio; struct xe_device *xe = gt_to_xe(gt); u64 lrc_desc; - u32 ring_mode = REG_MASKED_FIELD_ENABLE(GFX_DISABLE_LEGACY_MODE); lrc_desc = xe_lrc_descriptor(lrc); @@ -59,10 +58,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, lrc_desc |= FIELD_PREP(SW_CTX_ID, ctx_id); } - if (hwe->class == XE_ENGINE_CLASS_COMPUTE) - xe_mmio_write32(mmio, RCU_MODE, - REG_MASKED_FIELD_ENABLE(RCU_MODE_CCS_ENABLE)); - xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); lrc->ring.old_tail = lrc->ring.tail; @@ -82,10 +77,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, xe_bo_ggtt_addr(hwe->hwsp)); xe_mmio_read32(mmio, RING_HWS_PGA(hwe->mmio_base)); - if (xe_device_has_msix(gt_to_xe(hwe->gt))) - ring_mode |= REG_MASKED_FIELD_ENABLE(GFX_MSIX_INTERRUPT_ENABLE); - xe_mmio_write32(mmio, RING_MODE(hwe->mmio_base), ring_mode); - xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_LO(hwe->mmio_base), lower_32_bits(lrc_desc)); xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_HI(hwe->mmio_base), @@ -337,7 +328,6 @@ static int execlist_exec_queue_init(struct xe_exec_queue *q) struct drm_gpu_scheduler *sched; const struct drm_sched_init_args args = { .ops = &drm_sched_ops, - .num_rqs = 1, .credit_limit = xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, .hang_limit = XE_SCHED_HANG_LIMIT, .timeout = XE_SCHED_JOB_TIMEOUT, diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index a848d1a41b9b..a351c578b170 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -24,6 +24,7 @@ #include "xe_gt_types.h" #include "xe_map.h" #include "xe_mmio.h" +#include "xe_pat.h" #include "xe_pm.h" #include "xe_res_cursor.h" #include "xe_sriov.h" @@ -115,7 +116,6 @@ struct xe_ggtt { /** @size: Total usable size of this GGTT */ u64 size; -#define XE_GGTT_FLAGS_64K BIT(0) /** * @flags: Flags for this GGTT * Acceptable flags: @@ -259,7 +259,7 @@ static u64 xe_ggtt_get_pte(struct xe_ggtt *ggtt, u64 addr) static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size) { - u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), XE_CACHE_WB); u64 end = start + size - 1; u64 scratch_pte; @@ -724,7 +724,7 @@ static void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_ggtt_node *node, void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo) { u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB; - u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode]; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode); u64 pte; mutex_lock(&ggtt->lock); @@ -841,7 +841,7 @@ static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, bo->ggtt_node[tile_id] = NULL; } else { u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB; - u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode]; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode); u64 pte = ggtt->pt_ops->pte_encode_flags(bo, pat_index); xe_ggtt_map_bo(ggtt, bo->ggtt_node[tile_id], bo, pte); diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.c b/drivers/gpu/drm/xe/xe_gpu_scheduler.c index 9c8004d5dd91..67d8ce368486 100644 --- a/drivers/gpu/drm/xe/xe_gpu_scheduler.c +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.c @@ -66,7 +66,6 @@ int xe_sched_init(struct xe_gpu_scheduler *sched, const struct drm_sched_init_args args = { .ops = ops, .submit_wq = submit_wq, - .num_rqs = 1, .credit_limit = hw_submission, .hang_limit = hang_limit, .timeout = timeout, diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 8a31c963c372..783eb6d631b5 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -7,6 +7,8 @@ #include <linux/minmax.h> +#include <kunit/visibility.h> + #include <drm/drm_managed.h> #include <uapi/drm/xe_drm.h> @@ -393,10 +395,7 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt) if (gt->default_lrc[hwe->class]) continue; - xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe); - xe_wa_process_lrc(hwe); - xe_hw_engine_setup_default_lrc_state(hwe); - xe_tuning_process_lrc(hwe); + xe_hw_engine_setup_reg_lrc(hwe); default_lrc = drmm_kzalloc(&xe->drm, xe_gt_lrc_size(gt, hwe->class), @@ -788,6 +787,7 @@ void xe_gt_mmio_init(struct xe_gt *gt) if (IS_SRIOV_VF(xe)) gt->mmio.sriov_vf_gt = gt; } +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mmio_init); void xe_gt_record_user_engines(struct xe_gt *gt) { @@ -993,7 +993,6 @@ void xe_gt_reset_async(struct xe_gt *gt) void xe_gt_suspend_prepare(struct xe_gt *gt) { - CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); xe_uc_suspend_prepare(>->uc); } diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h index de7e47763411..4150aa594f05 100644 --- a/drivers/gpu/drm/xe/xe_gt.h +++ b/drivers/gpu/drm/xe/xe_gt.h @@ -155,4 +155,19 @@ static inline bool xe_gt_recovery_pending(struct xe_gt *gt) xe_gt_sriov_vf_recovery_pending(gt); } +/** + * xe_gt_supports_multi_queue() - Check if gt supports multi queue for the + * specified engine class. + * + * @gt: the GT object + * @class: hwe class type + * + * Return: true if the hw engine class supports multi queue, else false + */ +static inline bool xe_gt_supports_multi_queue(const struct xe_gt *gt, + enum xe_engine_class class) +{ + return gt->info.multi_queue_engine_class_mask & BIT(class); +} + #endif diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index f45306308cd6..c38bcacb27e4 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -149,8 +149,10 @@ static int register_save_restore(struct xe_gt *gt, struct drm_printer *p) drm_printf(p, "\n"); drm_printf(p, "Whitelist\n"); - for_each_hw_engine(hwe, gt, id) + for_each_hw_engine(hwe, gt, id) { xe_reg_whitelist_dump(&hwe->reg_whitelist, p); + xe_reg_whitelist_dump(&hwe->oa_whitelist, p); + } return 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index 4a2d9edb6a4c..04b24e1c8b78 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -3,6 +3,8 @@ * Copyright © 2023 Intel Corporation */ +#include <linux/time64.h> + #include <drm/drm_managed.h> #include <generated/xe_wa_oob.h> @@ -93,7 +95,7 @@ static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency) gtidle->cur_residency = cur_residency; /* residency multiplier in ns, convert to ms */ - cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6); + cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, NSEC_PER_MSEC); return cur_residency; } diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c index 7c6f039c880d..04f0098070a4 100644 --- a/drivers/gpu/drm/xe/xe_gt_mcr.c +++ b/drivers/gpu/drm/xe/xe_gt_mcr.c @@ -3,6 +3,9 @@ * Copyright © 2022 Intel Corporation */ +#include <kunit/static_stub.h> +#include <kunit/visibility.h> + #include "xe_gt_mcr.h" #include "regs/xe_gt_regs.h" @@ -216,9 +219,7 @@ static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = { static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = { { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */ - { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */ { 0x00C800, 0x00CFFF }, /* GAM */ - { 0x00D880, 0x00D8FF }, /* NODE */ { 0x00DD00, 0x00DDFF }, /* MEMPIPE */ { 0x00E900, 0x00E97F }, /* MEMPIPE */ { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ @@ -267,7 +268,7 @@ static const struct xe_mmio_range xe3p_xpc_gam_grp1_steering_table[] = { {}, }; -static const struct xe_mmio_range xe3p_xpc_node_steering_table[] = { +static const struct xe_mmio_range xe2_node_steering_table[] = { { 0x00B000, 0x00B0FF }, { 0x00D880, 0x00D8FF }, {}, @@ -298,7 +299,7 @@ static void init_steering_l3bank(struct xe_gt *gt) struct xe_device *xe = gt_to_xe(gt); struct xe_mmio *mmio = >->mmio; - if (GRAPHICS_VER(xe) >= 35) { + if (GRAPHICS_VER(xe) >= 20) { unsigned int first_bank = xe_l3_bank_mask_ffs(gt->fuse_topo.l3_bank_mask); const int banks_per_node = 4; unsigned int node = first_bank / banks_per_node; @@ -430,19 +431,6 @@ void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *g *instance = dss % gt->steering_dss_per_grp; } -/** - * xe_gt_mcr_steering_info_to_dss_id - Get DSS ID from group/instance steering - * @gt: GT structure - * @group: steering group ID - * @instance: steering instance ID - * - * Return: the converted DSS id. - */ -u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance) -{ - return group * dss_per_group(gt) + instance; -} - static void init_steering_dss(struct xe_gt *gt) { gt->steering_dss_per_grp = dss_per_group(gt); @@ -536,7 +524,7 @@ void xe_gt_mcr_init_early(struct xe_gt *gt) gt->steering[GAM1].ranges = xe3p_xpc_gam_grp1_steering_table; gt->steering[INSTANCE0].ranges = xe3p_xpc_instance0_steering_table; gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; - gt->steering[NODE].ranges = xe3p_xpc_node_steering_table; + gt->steering[NODE].ranges = xe2_node_steering_table; } else if (GRAPHICS_VERx100(xe) >= 3510) { gt->steering[DSS].ranges = xe2lpg_dss_steering_table; gt->steering[INSTANCE0].ranges = xe3p_lpg_instance0_steering_table; @@ -544,6 +532,8 @@ void xe_gt_mcr_init_early(struct xe_gt *gt) gt->steering[DSS].ranges = xe2lpg_dss_steering_table; gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table; gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table; + gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; + gt->steering[NODE].ranges = xe2_node_steering_table; } else if (GRAPHICS_VERx100(xe) >= 1270) { gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; @@ -566,6 +556,7 @@ void xe_gt_mcr_init_early(struct xe_gt *gt) /* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */ gt->steering[INSTANCE0].initialized = true; } +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_init_early); /** * xe_gt_mcr_init - Normal initialization of the MCR support @@ -613,6 +604,40 @@ void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) } } +static bool reg_in_steering_type_ranges(struct xe_gt *gt, + struct xe_reg reg, + int type) +{ + if (!gt->steering[type].ranges) + return false; + + for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) + if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) + return true; + + return false; +} + +/* + * xe_gt_mcr_check_reg - check if a register is recognized by this GT as MCR + * @gt: GT structure + * @reg: The register to check + * + * Returns true if the register offset falls within one of the MMIO ranges + * classified as MCR for the GT. + */ +bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg) +{ + KUNIT_STATIC_STUB_REDIRECT(xe_gt_mcr_check_reg, gt, reg); + + for (int type = 0; type <= IMPLICIT_STEERING; type++) + if (reg_in_steering_type_ranges(gt, reg, type)) + return true; + + return false; +} +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_check_reg); + /* * xe_gt_mcr_get_nonterminated_steering - find group/instance values that * will steer a register to a non-terminated instance @@ -634,30 +659,21 @@ bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, u8 *group, u8 *instance) { const struct xe_reg reg = to_xe_reg(reg_mcr); - const struct xe_mmio_range *implicit_ranges; for (int type = 0; type < IMPLICIT_STEERING; type++) { - if (!gt->steering[type].ranges) - continue; - - for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) { - if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) { - drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized, - "Uninitialized usage of MCR register %s/%#x\n", - xe_steering_types[type].name, reg.addr); - - *group = gt->steering[type].group_target; - *instance = gt->steering[type].instance_target; - return true; - } + if (reg_in_steering_type_ranges(gt, reg, type)) { + drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized, + "Uninitialized usage of MCR register %s/%#x\n", + xe_steering_types[type].name, reg.addr); + + *group = gt->steering[type].group_target; + *instance = gt->steering[type].instance_target; + return true; } } - implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges; - if (implicit_ranges) - for (int i = 0; implicit_ranges[i].end > 0; i++) - if (xe_mmio_in_range(>->mmio, &implicit_ranges[i], reg)) - return false; + if (reg_in_steering_type_ranges(gt, reg, IMPLICIT_STEERING)) + return false; /* * Not found in a steering table and not a register with implicit diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.h b/drivers/gpu/drm/xe/xe_gt_mcr.h index 283a1c9770e2..75374662f10d 100644 --- a/drivers/gpu/drm/xe/xe_gt_mcr.h +++ b/drivers/gpu/drm/xe/xe_gt_mcr.h @@ -26,6 +26,7 @@ void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg, void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg, u32 value); +bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg); bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, u8 *group, u8 *instance); @@ -33,7 +34,6 @@ bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p); void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance); -u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance); /* * Loop over each DSS and determine the group and instance IDs that diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c index fb5c9101e275..7868056841b3 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c @@ -219,6 +219,7 @@ static void pf_restart(struct xe_gt *gt) xe_gt_assert(gt, !xe_pm_runtime_suspended(xe)); + xe_gt_sriov_pf_policy_restart(gt); xe_gt_sriov_pf_config_restart(gt); xe_gt_sriov_pf_control_restart(gt); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 2f376b5fb088..2c9b85b84b1b 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -285,6 +285,13 @@ static u32 encode_config_ggtt(u32 *cfg, const struct xe_gt_sriov_config *config, return encode_ggtt(cfg, xe_ggtt_node_addr(node), xe_ggtt_node_size(node), details); } +static bool custom_sched_priority(struct xe_gt *gt, u32 priority) +{ + return xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt) ? + priority != GUC_SCHED_PRIORITY_NORMAL : + priority != GUC_SCHED_PRIORITY_LOW; +} + static u32 encode_config_sched(struct xe_gt *gt, u32 *cfg, u32 n, const struct xe_gt_sriov_config *config) { @@ -313,6 +320,11 @@ static u32 encode_config_sched(struct xe_gt *gt, u32 *cfg, u32 n, cfg[n++] = config->preempt_timeout[0]; } + if (custom_sched_priority(gt, config->sched_priority)) { + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_SCHED_PRIORITY); + cfg[n++] = config->sched_priority; + } + return n; } @@ -1922,29 +1934,45 @@ static u64 pf_profile_fair_lmem(struct xe_gt *gt, unsigned int num_vfs) return ALIGN_DOWN(fair, alignment); } -static void __pf_show_provisioning_lmem(struct xe_gt *gt, unsigned int first_vf, - unsigned int num_vfs, bool provisioned) +static void __pf_show_provisioned(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned, + u32 (*get32)(struct xe_gt *, unsigned int), + u64 (*get64)(struct xe_gt *, unsigned int), + const char *what) { unsigned int allvfs = 1 + xe_gt_sriov_pf_get_totalvfs(gt); /* PF plus VFs */ unsigned long *bitmap __free(bitmap) = bitmap_zalloc(allvfs, GFP_KERNEL); unsigned int weight; unsigned int n; + bool pf; + + xe_gt_assert(gt, get32 || get64); if (!bitmap) return; for (n = first_vf; n < first_vf + num_vfs; n++) { - if (!!pf_get_vf_config_lmem(gt, VFID(n)) == provisioned) + if ((get32 && (!!get32(gt, VFID(n)) == provisioned)) || + (get64 && (!!get64(gt, VFID(n)) == provisioned))) bitmap_set(bitmap, n, 1); } + pf = test_and_clear_bit(0, bitmap); weight = bitmap_weight(bitmap, allvfs); - if (!weight) + if (!pf && !weight) return; - xe_gt_sriov_info(gt, "VF%s%*pbl %s provisioned with VRAM\n", - weight > 1 ? "s " : "", allvfs, bitmap, - provisioned ? "already" : "not"); + xe_gt_sriov_info(gt, "%s%s%s%s%*pbl %s provisioned with %s\n", + pf ? "PF" : "", pf && weight ? " and " : "", + weight ? "VF" : "", weight > 1 ? "s " : "", + allvfs, bitmap, provisioned ? "already" : "not", what); +} + +static void __pf_show_provisioning_lmem(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned) +{ + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + NULL, pf_get_vf_config_lmem, "VRAM"); } static void pf_show_all_provisioned_lmem(struct xe_gt *gt) @@ -2078,6 +2106,17 @@ static u32 pf_get_exec_quantum(struct xe_gt *gt, unsigned int vfid) return config->exec_quantum[0]; } +static bool pf_non_default_exec_quantum(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int i; + + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) + if (config->exec_quantum[i]) + return true; + return false; +} + /** * xe_gt_sriov_pf_config_set_exec_quantum_locked() - Configure PF/VF execution quantum. * @gt: the &xe_gt @@ -2154,34 +2193,41 @@ u32 xe_gt_sriov_pf_config_get_exec_quantum(struct xe_gt *gt, unsigned int vfid) return pf_get_exec_quantum(gt, vfid); } -/** - * xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked() - Configure EQ for PF and VFs. - * @gt: the &xe_gt to configure - * @exec_quantum: requested execution quantum in milliseconds (0 is infinity) - * - * This function can only be called on PF with the master mutex hold. - * - * Return: 0 on success or a negative error code on failure. - */ -int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum) +static int pf_bulk_set_exec_quantum(struct xe_gt *gt, u32 exec_quantum, + unsigned int first_vf, unsigned int num_vfs) { - unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); unsigned int n; int err = 0; lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); - for (n = 0; n <= totalvfs; n++) { + for (n = first_vf; n < first_vf + num_vfs; n++) { err = pf_provision_exec_quantum(gt, VFID(n), exec_quantum); if (err) break; } - return pf_config_bulk_set_u32_done(gt, 0, 1 + totalvfs, exec_quantum, + return pf_config_bulk_set_u32_done(gt, first_vf, num_vfs, exec_quantum, pf_get_exec_quantum, "execution quantum", exec_quantum_unit, n, err); } +/** + * xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked() - Configure EQ for PF and VFs. + * @gt: the &xe_gt to configure + * @exec_quantum: requested execution quantum in milliseconds (0 is infinity) + * + * This function can only be called on PF with the master mutex hold. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum) +{ + unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); + + return pf_bulk_set_exec_quantum(gt, exec_quantum, PFID, 1 + totalvfs); +} + static int pf_provision_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, const u32 *exec_quantums, u32 count) { @@ -2293,6 +2339,17 @@ static u32 pf_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid) return config->preempt_timeout[0]; } +static bool pf_non_default_preempt_timeout(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int i; + + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) + if (config->preempt_timeout[i]) + return true; + return false; +} + /** * xe_gt_sriov_pf_config_set_preempt_timeout_locked() - Configure PF/VF preemption timeout. * @gt: the &xe_gt @@ -2368,34 +2425,41 @@ u32 xe_gt_sriov_pf_config_get_preempt_timeout(struct xe_gt *gt, unsigned int vfi return pf_get_preempt_timeout(gt, vfid); } -/** - * xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked() - Configure PT for PF and VFs. - * @gt: the &xe_gt to configure - * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity) - * - * This function can only be called on PF with the master mutex hold. - * - * Return: 0 on success or a negative error code on failure. - */ -int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout) +static int pf_bulk_set_preempt_timeout(struct xe_gt *gt, u32 preempt_timeout, + unsigned int first_vf, unsigned int num_vfs) { - unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); unsigned int n; int err = 0; lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); - for (n = 0; n <= totalvfs; n++) { + for (n = first_vf; n < first_vf + num_vfs; n++) { err = pf_provision_preempt_timeout(gt, VFID(n), preempt_timeout); if (err) break; } - return pf_config_bulk_set_u32_done(gt, 0, 1 + totalvfs, preempt_timeout, + return pf_config_bulk_set_u32_done(gt, first_vf, num_vfs, preempt_timeout, pf_get_preempt_timeout, "preemption timeout", preempt_timeout_unit, n, err); } +/** + * xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked() - Configure PT for PF and VFs. + * @gt: the &xe_gt to configure + * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity) + * + * This function can only be called on PF with the master mutex hold. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout) +{ + unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); + + return pf_bulk_set_preempt_timeout(gt, preempt_timeout, PFID, 1 + totalvfs); +} + static int pf_provision_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, const u32 *preempt_timeouts, u32 count) { @@ -2500,7 +2564,7 @@ static int pf_provision_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 return 0; } -static int pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid) +static u32 pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid) { struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); @@ -2550,6 +2614,35 @@ u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid return priority; } +/** + * xe_gt_sriov_pf_config_force_sched_priority_locked() - Force update scheduling priority. + * @gt: the &xe_gt + * @priority: new scheduling priority to set + * + * This function allows to update cached values of the scheduling priorities of all + * VFs (and PF) as result of applying the `GUC_KLV_VGT_POLICY_SCHED_IF_IDLE`_ policy. + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority) +{ + unsigned int total_vfs = xe_gt_sriov_pf_get_totalvfs(gt); + struct xe_gt_sriov_config *config; + unsigned int n; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = 0; n <= total_vfs; n++) { + config = pf_pick_vf_config(gt, VFID(n)); + config->sched_priority = priority; + } + + pf_config_bulk_set_u32_done(gt, PFID, 1 + total_vfs, priority, + pf_get_sched_priority, "scheduling priority", + sched_priority_unit, n, 0); +} + static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *config) { int i; @@ -2562,6 +2655,104 @@ static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *c } } +static bool pf_non_default_sched(struct xe_gt *gt, unsigned int vfid) +{ + return pf_non_default_exec_quantum(gt, vfid) || + pf_non_default_preempt_timeout(gt, vfid) || + custom_sched_priority(gt, pf_get_sched_priority(gt, vfid)); +} + +static void __pf_show_provisioned_sched(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned) +{ + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + pf_get_exec_quantum, NULL, "EQ"); + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + pf_get_preempt_timeout, NULL, "PT"); + + /* we only care about non-default priorities */ + if (provisioned) + __pf_show_provisioned(gt, first_vf, num_vfs, true, + pf_get_sched_priority, NULL, "PRIORITY"); +} + +static void pf_show_all_provisioned_sched(struct xe_gt *gt) +{ + __pf_show_provisioned_sched(gt, PFID, 1 + xe_gt_sriov_pf_get_totalvfs(gt), true); +} + +static void pf_show_unprovisioned_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + __pf_show_provisioned_sched(gt, PFID, 1 + num_vfs, false); +} + +static bool pf_needs_provision_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + unsigned int vfid; + + for (vfid = PFID; vfid <= PFID + num_vfs; vfid++) { + if (pf_non_default_sched(gt, vfid)) { + pf_show_all_provisioned_sched(gt); + pf_show_unprovisioned_sched(gt, num_vfs); + return false; + } + } + + if (xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt)) { + pf_show_all_provisioned_sched(gt); + pf_show_unprovisioned_sched(gt, num_vfs); + return false; + } + + pf_show_all_provisioned_sched(gt); + return true; +} + +/* With 16ms EQ/PT GuC should be able to handle up to 63 VFs within 2s */ +#define XE_FAIR_EXEC_QUANTUM_MS 16 +#define XE_FAIR_PREEMPT_TIMEOUT_US 16000 +#define XE_FAIR_SCHED_PRIORITY GUC_SCHED_PRIORITY_LOW +#define XE_ADMIN_PF_SCHED_PRIORITY GUC_SCHED_PRIORITY_HIGH + +/** + * xe_gt_sriov_pf_config_set_fair_sched() - Provision PF and VFs with fair scheduling. + * @gt: the &xe_gt + * @num_vfs: number of VFs to provision (can't be 0) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + int result = 0; + int err; + + xe_gt_assert(gt, num_vfs); + xe_gt_assert(gt, XE_FAIR_EXEC_QUANTUM_MS); + xe_gt_assert(gt, XE_FAIR_PREEMPT_TIMEOUT_US); + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + if (!pf_needs_provision_sched(gt, num_vfs)) + return 0; + + err = pf_bulk_set_exec_quantum(gt, XE_FAIR_EXEC_QUANTUM_MS, PFID, 1 + num_vfs); + result = result ?: err; + err = pf_bulk_set_preempt_timeout(gt, XE_FAIR_PREEMPT_TIMEOUT_US, PFID, 1 + num_vfs); + result = result ?: err; + + xe_gt_assert(gt, XE_FAIR_SCHED_PRIORITY == GUC_SCHED_PRIORITY_LOW); + xe_gt_assert(gt, !xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt)); + + if (xe_sriov_pf_admin_only(gt_to_xe(gt))) { + err = pf_provision_sched_priority(gt, PFID, XE_ADMIN_PF_SCHED_PRIORITY); + result = result ?: err; + } + + return result; +} + static int pf_provision_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index, u32 value) { @@ -2838,6 +3029,9 @@ static int pf_validate_vf_config(struct xe_gt *gt, unsigned int vfid) valid_all = valid_all && valid_lmem; } + /* also check optional EQ/PT/PRIO */ + valid_any = valid_any || pf_non_default_sched(gt, vfid); + return valid_all ? 0 : valid_any ? -ENOKEY : -ENODATA; } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h index 4a004ecd6140..2ec62c12ad5c 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h @@ -71,12 +71,14 @@ int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid); int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority); +void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority); u32 xe_gt_sriov_pf_config_get_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index); int xe_gt_sriov_pf_config_set_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index, u32 value); +int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs); int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout); int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c index 848e24926ecd..e8458d63742d 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c @@ -8,6 +8,7 @@ #include "abi/guc_actions_sriov_abi.h" #include "xe_gt.h" +#include "xe_gt_sriov_pf_config.h" #include "xe_gt_sriov_pf_helpers.h" #include "xe_gt_sriov_pf_policy.h" #include "xe_gt_sriov_printk.h" @@ -16,7 +17,6 @@ #include "xe_guc_ct.h" #include "xe_guc_klv_helpers.h" #include "xe_guc_submit.h" -#include "xe_pm.h" /* * Return: number of KLVs that were successfully parsed and saved, @@ -67,6 +67,15 @@ static int pf_push_policy_buf_klvs(struct xe_gt *gt, u32 num_klvs, return err; } + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) { + struct drm_printer p = xe_gt_dbg_printer(gt); + void *klvs = xe_guc_buf_cpu_ptr(buf); + + xe_gt_sriov_dbg(gt, "pushed policy update with %u KLV%s:\n", + num_klvs, str_plural(num_klvs)); + xe_guc_klv_print(klvs, num_dwords, &p); + } + return 0; } @@ -153,33 +162,14 @@ static int pf_update_policy_u32(struct xe_gt *gt, u16 key, u32 *policy, u32 valu return 0; } -static void pf_bulk_reset_sched_priority(struct xe_gt *gt, u32 priority) -{ - unsigned int total_vfs = 1 + xe_gt_sriov_pf_get_totalvfs(gt); - unsigned int n; - - xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); - lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); - - for (n = 0; n < total_vfs; n++) - gt->sriov.pf.vfs[n].config.sched_priority = priority; -} - static int pf_provision_sched_if_idle(struct xe_gt *gt, bool enable) { - int err; - xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); - err = pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY, - >->sriov.pf.policy.guc.sched_if_idle, - enable); - - if (!err) - pf_bulk_reset_sched_priority(gt, enable ? GUC_SCHED_PRIORITY_NORMAL : - GUC_SCHED_PRIORITY_LOW); - return err; + return pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY, + >->sriov.pf.policy.guc.sched_if_idle, + enable); } static int pf_reprovision_sched_if_idle(struct xe_gt *gt) @@ -187,6 +177,9 @@ static int pf_reprovision_sched_if_idle(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + if (!gt->sriov.pf.policy.guc.sched_if_idle) + return 0; + return pf_provision_sched_if_idle(gt, gt->sriov.pf.policy.guc.sched_if_idle); } @@ -199,7 +192,7 @@ static void pf_sanitize_sched_if_idle(struct xe_gt *gt) } /** - * xe_gt_sriov_pf_policy_set_sched_if_idle - Control the 'sched_if_idle' policy. + * xe_gt_sriov_pf_policy_set_sched_if_idle_locked() - Control the 'sched_if_idle' policy. * @gt: the &xe_gt where to apply the policy * @enable: the value of the 'sched_if_idle' policy * @@ -207,15 +200,60 @@ static void pf_sanitize_sched_if_idle(struct xe_gt *gt) * * Return: 0 on success or a negative error code on failure. */ -int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable) +int xe_gt_sriov_pf_policy_set_sched_if_idle_locked(struct xe_gt *gt, bool enable) { + u32 priority; int err; - mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_provision_sched_if_idle(gt, enable); - mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + if (err) + return err; - return err; + /* + * As of GuC 70.12 a change of this policy impacts individual configs + * of all VFs. See `GUC_KLV_VGT_POLICY_SCHED_IF_IDLE`_ for details. + */ + xe_gt_assert(gt, GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 12)); + + priority = enable ? GUC_SCHED_PRIORITY_NORMAL : GUC_SCHED_PRIORITY_LOW; + xe_gt_sriov_pf_config_force_sched_priority_locked(gt, priority); + + return 0; +} + +/** + * xe_gt_sriov_pf_policy_set_sched_if_idle() - Control the 'sched_if_idle' policy. + * @gt: the &xe_gt where to apply the policy + * @enable: the value of the 'sched_if_idle' policy + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, enable); +} + +/** + * xe_gt_sriov_pf_policy_get_sched_if_idle_locked() - Retrieve value of 'sched_if_idle' policy. + * @gt: the &xe_gt where to read the policy from + * + * This function can only be called on PF. + * + * Return: last value of 'sched_if_idle' policy applied. + */ +bool xe_gt_sriov_pf_policy_get_sched_if_idle_locked(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return gt->sriov.pf.policy.guc.sched_if_idle; } /** @@ -228,15 +266,10 @@ int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable) */ bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt) { - bool enable; - xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); - mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); - enable = gt->sriov.pf.policy.guc.sched_if_idle; - mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); - - return enable; + return xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt); } static int pf_provision_reset_engine(struct xe_gt *gt, bool enable) @@ -253,6 +286,9 @@ static int pf_reprovision_reset_engine(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + if (!gt->sriov.pf.policy.guc.reset_engine) + return 0; + return pf_provision_reset_engine(gt, gt->sriov.pf.policy.guc.reset_engine); } @@ -319,6 +355,9 @@ static int pf_reprovision_sample_period(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + if (!gt->sriov.pf.policy.guc.sample_period) + return 0; + return pf_provision_sample_period(gt, gt->sriov.pf.policy.guc.sample_period); } @@ -689,30 +728,23 @@ void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt) } /** - * xe_gt_sriov_pf_policy_reprovision - Reprovision (and optionally reset) policy settings. + * xe_gt_sriov_pf_policy_restart() - Reprovision policy settings. * @gt: the &xe_gt - * @reset: if true will reprovision using default values instead of latest * * This function can only be called on PF. * * Return: 0 on success or a negative error code on failure. */ -int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset) +int xe_gt_sriov_pf_policy_restart(struct xe_gt *gt) { int err = 0; - xe_pm_runtime_get_noresume(gt_to_xe(gt)); + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); - mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); - if (reset) - pf_sanitize_guc_policies(gt); err |= pf_reprovision_sched_if_idle(gt); err |= pf_reprovision_reset_engine(gt); err |= pf_reprovision_sample_period(gt); err |= pf_reprovision_sched_groups(gt); - mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); - - xe_pm_runtime_put(gt_to_xe(gt)); return err ? -ENXIO : 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h index bd73aa58f9ca..d667d66897b1 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h @@ -14,7 +14,9 @@ struct drm_printer; struct xe_gt; int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable); +int xe_gt_sriov_pf_policy_set_sched_if_idle_locked(struct xe_gt *gt, bool enable); bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt); +bool xe_gt_sriov_pf_policy_get_sched_if_idle_locked(struct xe_gt *gt); int xe_gt_sriov_pf_policy_set_reset_engine(struct xe_gt *gt, bool enable); bool xe_gt_sriov_pf_policy_get_reset_engine(struct xe_gt *gt); int xe_gt_sriov_pf_policy_set_sample_period(struct xe_gt *gt, u32 value); @@ -29,7 +31,7 @@ bool xe_gt_sriov_pf_policy_sched_groups_enabled(struct xe_gt *gt); void xe_gt_sriov_pf_policy_init(struct xe_gt *gt); void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt); -int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset); +int xe_gt_sriov_pf_policy_restart(struct xe_gt *gt); int xe_gt_sriov_pf_policy_print(struct xe_gt *gt, struct drm_printer *p); #endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h index 667b8310478d..6296483c44ec 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h @@ -49,7 +49,6 @@ struct xe_gt_sriov_pf_workers { * @service: service data. * @control: control data. * @policy: policy data. - * @migration: migration data. * @spare: PF-only provisioning configuration. * @vfs: metadata for all VFs. */ diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c index 59b3b23a54c8..789397514f3e 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats.c +++ b/drivers/gpu/drm/xe/xe_gt_stats.c @@ -9,6 +9,47 @@ #include "xe_device.h" #include "xe_gt_stats.h" +/** + * DOC: Xe GT Statistics + * + * Overview + * ======== + * + * The Xe driver exposes per-GT statistics through the debugfs filesystem at:: + * + * /sys/kernel/debug/dri/<device>/gt<id>/stats + * + * This interface requires the kernel to be built with ``CONFIG_DEBUG_FS=y``. + * + * Reading statistics + * ================== + * + * Reading the file prints all available statistics, one per line, in + * ``name: value`` format:: + * + * $ cat /sys/kernel/debug/dri/0/gt0/stats + * svm_pagefault_count: 0 + * tlb_inval_count: 1234 + * ... + * + * All values are 64-bit unsigned integers aggregated across all CPUs. + * Counters accumulate since the driver was loaded or since the last explicit + * reset. Timing counters use microseconds as their unit; data volume counters + * use KiB. + * + * Resetting statistics + * ==================== + * + * Writing a boolean true value to the file resets all counters to zero:: + * + * echo 1 > /sys/kernel/debug/dri/0/gt0/stats + * + * Any value accepted by ``kstrtobool()`` (e.g. ``1``, ``y``, ``yes``, + * ``on``) triggers the reset. Resetting while the GPU is active may yield + * unpredictable intermediate values; it is recommended to reset only when + * the GPU is idle. + */ + static void xe_gt_stats_fini(struct drm_device *drm, void *arg) { struct xe_gt *gt = arg; diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h index 081c787ddcb6..425491bed6c4 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats_types.h +++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h @@ -8,6 +8,124 @@ #include <linux/types.h> +/** + * enum xe_gt_stats_id - GT statistics identifiers + * @XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT: Total SVM page faults handled. + * @XE_GT_STATS_ID_TLB_INVAL: Total GPU Translation Lookaside Buffer (TLB) + * invalidations issued. + * @XE_GT_STATS_ID_SVM_TLB_INVAL_COUNT: TLB invalidations issued during SVM + * page-fault handling. + * @XE_GT_STATS_ID_SVM_TLB_INVAL_US: Cumulative time (µs) waiting for TLB + * invalidations during SVM page-fault handling. + * + * @XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT: Buffer-object (non-SVM) page faults + * handled. + * @XE_GT_STATS_ID_VMA_PAGEFAULT_KB: Size (KiB) of VMAs involved in + * buffer-object page fault handling. + * @XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT: GPU prefetch faults for + * addresses with no valid backing. + * + * @XE_GT_STATS_ID_SVM_4K_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 4K pages. + * @XE_GT_STATS_ID_SVM_64K_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 64K pages. + * @XE_GT_STATS_ID_SVM_2M_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 2M pages. + * @XE_GT_STATS_ID_SVM_4K_VALID_PAGEFAULT_COUNT: Valid SVM page faults + * at 4K page size, where the GPU mapping was already valid — resolved without + * creating new mappings. + * @XE_GT_STATS_ID_SVM_64K_VALID_PAGEFAULT_COUNT: Valid SVM page faults at 64K + * page size. + * @XE_GT_STATS_ID_SVM_2M_VALID_PAGEFAULT_COUNT: Valid SVM page faults at 2M + * page size. + * @XE_GT_STATS_ID_SVM_4K_PAGEFAULT_US: Cumulative time (µs) handling 4K SVM + * page faults. + * @XE_GT_STATS_ID_SVM_64K_PAGEFAULT_US: Cumulative time (µs) handling 64K + * SVM page faults. + * @XE_GT_STATS_ID_SVM_2M_PAGEFAULT_US: Cumulative time (µs) handling 2M SVM + * page faults. + * + * @XE_GT_STATS_ID_SVM_4K_MIGRATE_COUNT: 4K pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_64K_MIGRATE_COUNT: 64K pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_2M_MIGRATE_COUNT: 2M pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_4K_MIGRATE_US: Cumulative time (µs) moving 4K pages + * from CPU to device memory. + * @XE_GT_STATS_ID_SVM_64K_MIGRATE_US: Cumulative time (µs) moving 64K pages + * from CPU to device memory. + * @XE_GT_STATS_ID_SVM_2M_MIGRATE_US: Cumulative time (µs) moving 2M pages + * from CPU to device memory. + * + * @XE_GT_STATS_ID_SVM_DEVICE_COPY_US: Cumulative time (µs) for memory copies to + * device, across all page sizes. + * @XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_US: Cumulative time (µs) for memory copies + * of 4K pages to device. + * @XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_US: Cumulative time (µs) for memory + * copies of 64K pages to device. + * @XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_US: Cumulative time (µs) for memory copies + * of 2M pages to device. + * @XE_GT_STATS_ID_SVM_CPU_COPY_US: Cumulative time (µs) for memory copies to + * CPU, across all page sizes. + * @XE_GT_STATS_ID_SVM_4K_CPU_COPY_US: Cumulative time (µs) for memory copies of + * 4K pages to CPU. + * @XE_GT_STATS_ID_SVM_64K_CPU_COPY_US: Cumulative time (µs) for memory copies + * of 64K pages to CPU. + * @XE_GT_STATS_ID_SVM_2M_CPU_COPY_US: Cumulative time (µs) for memory copies of + * 2M pages to CPU. + * @XE_GT_STATS_ID_SVM_DEVICE_COPY_KB: Data (KiB) copied to device across all + * page sizes. + * @XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_KB: Data (KiB) copied to device for 4K + * pages. + * @XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_KB: Data (KiB) copied to device for + * 64K pages. + * @XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_KB: Data (KiB) copied to device for 2M + * pages. + * @XE_GT_STATS_ID_SVM_CPU_COPY_KB: Data (KiB) copied to CPU across all page + * sizes. + * @XE_GT_STATS_ID_SVM_4K_CPU_COPY_KB: Data (KiB) copied to CPU for 4K pages. + * @XE_GT_STATS_ID_SVM_64K_CPU_COPY_KB: Data (KiB) copied to CPU for 64K pages. + * @XE_GT_STATS_ID_SVM_2M_CPU_COPY_KB: Data (KiB) copied to CPU for 2M pages. + * + * @XE_GT_STATS_ID_SVM_4K_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 4K page size. + * @XE_GT_STATS_ID_SVM_64K_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 64K page size. + * @XE_GT_STATS_ID_SVM_2M_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 2M page size. + * @XE_GT_STATS_ID_SVM_4K_BIND_US: Cumulative time (µs) binding 4K pages + * into the GPU page table. + * @XE_GT_STATS_ID_SVM_64K_BIND_US: Cumulative time (µs) binding 64K pages + * into the GPU page table. + * @XE_GT_STATS_ID_SVM_2M_BIND_US: Cumulative time (µs) binding 2M pages + * into the GPU page table. + * + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT: Times the + * scheduler preempted a long-running (LR) GPU exec queue. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT: Times the scheduler + * skipped suspend because the system was idle. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT: Times the driver + * stalled waiting for prior GPU work to complete before scheduling more. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US: Cumulative time + * (µs) spent preempting long-running (LR) GPU exec queues. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US: Cumulative time (µs) + * stalled waiting for prior GPU work to complete. + * + * @XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT: 4K-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT: 64K-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT: 2M-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_ISSUED_COUNT: Times a page reclamation was issued. + * @XE_GT_STATS_ID_PRL_ABORTED_COUNT: Times the page reclaim process was + * aborted. + * + * @__XE_GT_STATS_NUM_IDS: Number of valid IDs; not a real counter. + * + * See Documentation/gpu/xe/xe_gt_stats.rst. + */ enum xe_gt_stats_id { XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT, XE_GT_STATS_ID_TLB_INVAL, diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index fffb5d631b69..e5588c88800a 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -136,7 +136,7 @@ struct xe_gt { /** @info.gmdid: raw GMD_ID value from hardware */ u32 gmdid; /** - * @multi_queue_engine_class_mask: Bitmask of engine classes with + * @info.multi_queue_engine_class_mask: Bitmask of engine classes with * multi queue support enabled. */ u16 multi_queue_engine_class_mask; @@ -362,7 +362,7 @@ struct xe_gt { /** @user_engines: engines present in GT and available to userspace */ struct { /** - * @user_engines.mask: like @info->engine_mask, but take in + * @user_engines.mask: like @info.engine_mask, but take in * consideration only engines available to userspace */ u64 mask; diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index ccebb437e37f..4023700ff2a9 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -98,7 +98,7 @@ static u32 guc_ctl_feature_flags(struct xe_guc *guc) if (xe_guc_using_main_gamctrl_queues(guc)) flags |= GUC_CTL_MAIN_GAMCTRL_QUEUES; - if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe) && xe_gt_is_media_type(guc_to_gt(guc))) + if (xe_device_is_l2_flush_optimized(xe) && xe_gt_is_media_type(guc_to_gt(guc))) flags |= GUC_CTL_ENABLE_L2FLUSH_OPT; return flags; @@ -780,6 +780,14 @@ int xe_guc_init(struct xe_guc *guc) if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 14, 0)) xe->info.has_page_reclaim_hw_assist = false; + /* Disable indirect_ring_state if missing GuC 70.53+ WA 14025515070. */ + if (gt->info.has_indirect_ring_state && + XE_GT_WA(gt, 14025515070) && + GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 26, 0)) { + gt->info.has_indirect_ring_state = 0; + xe_gt_notice(gt, "indirect ring state requires WA in GuC submit ver 1.26+\n"); + } + if (IS_SRIOV_VF(xe)) { ret = devm_add_action_or_reset(xe->drm.dev, vf_guc_fini_hw, guc); if (ret) @@ -1693,13 +1701,8 @@ void xe_guc_reset_wait(struct xe_guc *guc) void xe_guc_stop_prepare(struct xe_guc *guc) { - if (!IS_SRIOV_VF(guc_to_xe(guc))) { - int err; - - err = xe_guc_pc_stop(&guc->pc); - xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n", - ERR_PTR(err)); - } + if (!IS_SRIOV_VF(guc_to_xe(guc))) + xe_guc_pc_stop(&guc->pc); } void xe_guc_stop(struct xe_guc *guc) diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 5760251cb685..c98454545a85 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -360,6 +360,9 @@ static void guc_waklv_init(struct xe_guc_ads *ads) if (XE_GT_WA(gt, 14020001231)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT); + if (XE_GT_WA(gt, 14025515070) && GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 53)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG); size = guc_ads_waklv_size(ads) - remain; if (!size) @@ -739,10 +742,8 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, struct xe_reg reg; bool skip; } *e, extra_regs[] = { - { .reg = RING_MODE(hwe->mmio_base), }, { .reg = RING_HWS_PGA(hwe->mmio_base), }, { .reg = RING_IMR(hwe->mmio_base), }, - { .reg = RCU_MODE, .skip = hwe != hwe_rcs_reset_domain }, { .reg = CCS_MODE, .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) }, }; diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index 2f5816c78fba..1a019137ddf4 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -111,7 +111,7 @@ struct __guc_capture_parsed_output { { RING_TAIL(0), REG_32BIT, 0, 0, 0, "RING_TAIL"}, \ { RING_CTL(0), REG_32BIT, 0, 0, 0, "RING_CTL"}, \ { RING_MI_MODE(0), REG_32BIT, 0, 0, 0, "RING_MI_MODE"}, \ - { RING_MODE(0), REG_32BIT, 0, 0, 0, "RING_MODE"}, \ + { GFX_MODE(0), REG_32BIT, 0, 0, 0, "GFX_MODE"}, \ { RING_ESR(0), REG_32BIT, 0, 0, 0, "RING_ESR"}, \ { RING_EMR(0), REG_32BIT, 0, 0, 0, "RING_EMR"}, \ { RING_EIR(0), REG_32BIT, 0, 0, 0, "RING_EIR"}, \ @@ -461,8 +461,14 @@ static void guc_capture_alloc_steered_lists(struct xe_guc *guc) if (!list || guc->capture->extlists) return; - total = bitmap_weight(gt->fuse_topo.g_dss_mask, sizeof(gt->fuse_topo.g_dss_mask) * 8) * - guc_capture_get_steer_reg_num(guc_to_xe(guc)); + { + xe_dss_mask_t all_dss; + + total = bitmap_weighted_or(all_dss, gt->fuse_topo.g_dss_mask, + gt->fuse_topo.c_dss_mask, + XE_MAX_DSS_FUSE_BITS) * + guc_capture_get_steer_reg_num(guc_to_xe(guc)); + } if (!total) return; @@ -1841,12 +1847,6 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm str_yes_no(snapshot->kernel_reserved)); for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) { - /* - * FIXME: During devcoredump print we should avoid accessing the - * driver pointers for gt or engine. Printing should be done only - * using the snapshot captured. Here we are accessing the gt - * pointer. It should be fixed. - */ list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, type, capture_class, false); snapshot_print_by_list_order(snapshot, p, type, list); diff --git a/drivers/gpu/drm/xe/xe_guc_capture.h b/drivers/gpu/drm/xe/xe_guc_capture.h index 34d6fdc64f56..dca97d52b192 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.h +++ b/drivers/gpu/drm/xe/xe_guc_capture.h @@ -3,8 +3,8 @@ * Copyright © 2021-2024 Intel Corporation */ -#ifndef _XE_GUC_CAPTURE_H -#define _XE_GUC_CAPTURE_H +#ifndef _XE_GUC_CAPTURE_H_ +#define _XE_GUC_CAPTURE_H_ #include <linux/types.h> #include "abi/guc_capture_abi.h" diff --git a/drivers/gpu/drm/xe/xe_guc_capture_types.h b/drivers/gpu/drm/xe/xe_guc_capture_types.h index 6cb439115597..058a3f2eadce 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture_types.h +++ b/drivers/gpu/drm/xe/xe_guc_capture_types.h @@ -3,8 +3,8 @@ * Copyright © 2021-2024 Intel Corporation */ -#ifndef _XE_GUC_CAPTURE_TYPES_H -#define _XE_GUC_CAPTURE_TYPES_H +#ifndef _XE_GUC_CAPTURE_TYPES_H_ +#define _XE_GUC_CAPTURE_TYPES_H_ #include <linux/types.h> #include "regs/xe_reg_defs.h" @@ -32,7 +32,7 @@ struct __guc_mmio_reg_descr { /** * @data_type: data type of the register * Could be 32 bit, low or hi dword of a 64 bit, see enum - * register_data_type + * capture_register_data_type */ enum capture_register_data_type data_type; /** @flags: Flags for the register */ diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index a11cff7a20be..21e0dad9a481 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -186,13 +186,16 @@ static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { } struct g2h_fence { u32 *response_buffer; u32 seqno; + /* fields below this point are setup based on the response */ u32 response_data; u16 response_len; u16 error; u16 hint; u16 reason; + u32 counter; bool cancel; bool retry; + bool wait; bool fail; bool done; }; @@ -204,6 +207,11 @@ static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) g2h_fence->seqno = ~0x0; } +static void g2h_fence_reinit(struct g2h_fence *g2h_fence) +{ + memset_after(g2h_fence, 0, seqno); +} + static void g2h_fence_cancel(struct g2h_fence *g2h_fence) { g2h_fence->cancel = true; @@ -1331,6 +1339,7 @@ retry_same_fence: /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response * and g2h_fence_cancel. */ +wait_again: ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); if (!ret) { LNL_FLUSH_WORK(&ct->g2h_worker); @@ -1356,6 +1365,14 @@ retry_same_fence: return -ETIME; } + if (g2h_fence.wait) { + xe_gt_dbg(gt, "H2G action %#x busy: counter %u\n", + action[0], g2h_fence.counter); + /* we can't leave any response data if we want to wait again */ + g2h_fence_reinit(&g2h_fence); + mutex_unlock(&ct->lock); + goto wait_again; + } if (g2h_fence.retry) { xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", action[0], g2h_fence.reason); @@ -1508,7 +1525,12 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) return -EPROTO; } - g2h_fence = xa_erase(&ct->fence_lookup, fence); + /* don't erase as we still expect a final response with the same fence */ + if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) + g2h_fence = xa_load(&ct->fence_lookup, fence); + else + g2h_fence = xa_erase(&ct->fence_lookup, fence); + if (unlikely(!g2h_fence)) { /* Don't tear down channel, as send could've timed out */ /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ @@ -1519,6 +1541,12 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) xe_gt_assert(gt, fence == g2h_fence->seqno); + /* + * reinit as we might have already process this g2h_fence before + * if we received a NO_RESPONSE_BUSY reply + */ + g2h_fence_reinit(g2h_fence); + if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { g2h_fence->fail = true; g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); @@ -1526,6 +1554,9 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { g2h_fence->retry = true; g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); + } else if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) { + g2h_fence->wait = true; + g2h_fence->counter = FIELD_GET(GUC_HXG_BUSY_MSG_0_COUNTER, hxg[0]); } else if (g2h_fence->response_buffer) { g2h_fence->response_len = hxg_len; memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); @@ -1533,7 +1564,9 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); } - g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); + /* don't release any space if it was an intermediate message */ + if (!g2h_fence->wait) + g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ WRITE_ONCE(g2h_fence->done, true); @@ -1570,6 +1603,7 @@ static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) case GUC_HXG_TYPE_RESPONSE_SUCCESS: case GUC_HXG_TYPE_RESPONSE_FAILURE: case GUC_HXG_TYPE_NO_RESPONSE_RETRY: + case GUC_HXG_TYPE_NO_RESPONSE_BUSY: ret = parse_g2h_response(ct, msg, len); break; default: diff --git a/drivers/gpu/drm/xe/xe_guc_ct_types.h b/drivers/gpu/drm/xe/xe_guc_ct_types.h index 5da1ce5dc372..f88e588af0d3 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct_types.h +++ b/drivers/gpu/drm/xe/xe_guc_ct_types.h @@ -102,9 +102,9 @@ struct xe_dead_ct { bool reported; /** @worker: worker thread to get out of interrupt context before dumping */ struct work_struct worker; - /** snapshot_ct: copy of CT state and CTB content at point of error */ + /** @snapshot_ct: copy of CT state and CTB content at point of error */ struct xe_guc_ct_snapshot *snapshot_ct; - /** snapshot_log: copy of GuC log at point of error */ + /** @snapshot_log: copy of GuC log at point of error */ struct xe_guc_log_snapshot *snapshot_log; }; @@ -134,9 +134,9 @@ struct xe_guc_ct { spinlock_t fast_lock; /** @ctbs: buffers for sending and receiving commands */ struct { - /** @ctbs.send: Host to GuC (H2G, send) channel */ + /** @ctbs.h2g: Host to GuC (H2G, send) channel */ struct guc_ctb h2g; - /** @ctbs.recv: GuC to Host (G2H, receive) channel */ + /** @ctbs.g2h: GuC to Host (G2H, receive) channel */ struct guc_ctb g2h; } ctbs; /** @g2h_outstanding: number of outstanding G2H */ diff --git a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h index e5e53b421f29..cda14d954e57 100644 --- a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h @@ -10,6 +10,7 @@ #include <linux/workqueue.h> #include "xe_gpu_scheduler_types.h" +#include "xe_hw_fence_types.h" struct dma_fence; struct xe_exec_queue; @@ -24,6 +25,10 @@ struct xe_guc_exec_queue { struct rcu_head rcu; /** @sched: GPU scheduler for this xe_exec_queue */ struct xe_gpu_scheduler sched; + /** + * @name: Scheduler timeline name, kept with @sched until RCU free. + */ + char name[MAX_FENCE_NAME_LEN]; /** @entity: Scheduler entity for this xe_exec_queue */ struct xe_sched_entity entity; /** diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index b73fae063fac..3fbda4798cff 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_GUC_FWIF_H -#define _XE_GUC_FWIF_H +#ifndef _XE_GUC_FWIF_H_ +#define _XE_GUC_FWIF_H_ #include <linux/bits.h> diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index bb8c4e793492..59f2fa79ad42 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -890,9 +890,26 @@ void xe_guc_pc_init_early(struct xe_guc_pc *pc) pc_init_fused_rp_values(pc); } +static bool pc_needs_min_freq_change(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + + if (XE_DEVICE_WA(xe, 14022085890)) + return true; + + if (xe_gt_is_media_type(gt)) + return false; + + if (xe->info.platform == XE_BATTLEMAGE || + xe->info.platform == XE_CRESCENTISLAND) + return true; + + return false; +} + static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) { - struct xe_tile *tile = gt_to_tile(pc_to_gt(pc)); int ret; lockdep_assert_held(&pc->freq_lock); @@ -919,7 +936,18 @@ static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) if (pc_get_min_freq(pc) > pc->rp0_freq) ret = pc_set_min_freq(pc, pc->rp0_freq); - if (XE_DEVICE_WA(tile_to_xe(tile), 14022085890)) + /* + * Setting GT RP min frequency to 1.2GHz by default for + * GT0(Graphics) Tile of BMG and CRI. + * + * While BMG G21 WA will apply min frequency for + * both GT0(Graphics) and GT1(Media) Tile. + * + * This is an active frequency, so if the device is idle + * we aren't expecting high power output across board + * + */ + if (pc_needs_min_freq_change(pc)) ret = pc_set_min_freq(pc, max(BMG_MIN_FREQ, pc_get_min_freq(pc))); out: @@ -1288,18 +1316,16 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) * xe_guc_pc_stop - Stop GuC's Power Conservation component * @pc: Xe_GuC_PC instance */ -int xe_guc_pc_stop(struct xe_guc_pc *pc) +void xe_guc_pc_stop(struct xe_guc_pc *pc) { struct xe_device *xe = pc_to_xe(pc); if (xe->info.skip_guc_pc) - return 0; + return; mutex_lock(&pc->freq_lock); pc->freq_ready = false; mutex_unlock(&pc->freq_lock); - - return 0; } /** @@ -1314,10 +1340,10 @@ static void xe_guc_pc_fini_hw(void *arg) if (xe_device_wedged(xe)) return; - CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FW_GT); - XE_WARN_ON(xe_guc_pc_stop(pc)); + xe_guc_pc_stop(pc); /* Bind requested freq to mert_freq_cap before unload */ + CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FW_GT); pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc))); } diff --git a/drivers/gpu/drm/xe/xe_guc_pc.h b/drivers/gpu/drm/xe/xe_guc_pc.h index 0678a4e787b3..1025a2b15f5f 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.h +++ b/drivers/gpu/drm/xe/xe_guc_pc.h @@ -13,7 +13,7 @@ struct drm_printer; int xe_guc_pc_init(struct xe_guc_pc *pc); int xe_guc_pc_start(struct xe_guc_pc *pc); -int xe_guc_pc_stop(struct xe_guc_pc *pc); +void xe_guc_pc_stop(struct xe_guc_pc *pc); void xe_guc_pc_print(struct xe_guc_pc *pc, struct drm_printer *p); int xe_guc_pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value); int xe_guc_pc_action_unset_param(struct xe_guc_pc *pc, u8 id); diff --git a/drivers/gpu/drm/xe/xe_guc_relay.c b/drivers/gpu/drm/xe/xe_guc_relay.c index 577a315854af..eed0a750d2eb 100644 --- a/drivers/gpu/drm/xe/xe_guc_relay.c +++ b/drivers/gpu/drm/xe/xe_guc_relay.c @@ -689,12 +689,17 @@ static int relay_action_handler(struct xe_guc_relay *relay, u32 origin, return relay_testloop_action_handler(relay, origin, msg, len, response, size); type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); + relay_assert(relay, guc_hxg_type_is_action(type)); - if (IS_SRIOV_PF(relay_to_xe(relay))) - ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, response, size); - else + if (IS_SRIOV_PF(relay_to_xe(relay))) { + if (type == GUC_HXG_TYPE_REQUEST) + ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, + response, size); + else + ret = -EOPNOTSUPP; + } else { ret = -EOPNOTSUPP; - + } if (type == GUC_HXG_TYPE_EVENT) relay_assert(relay, ret <= 0); diff --git a/drivers/gpu/drm/xe/xe_guc_relay_types.h b/drivers/gpu/drm/xe/xe_guc_relay_types.h index 20eee10856b2..b99a6686416b 100644 --- a/drivers/gpu/drm/xe/xe_guc_relay_types.h +++ b/drivers/gpu/drm/xe/xe_guc_relay_types.h @@ -15,7 +15,7 @@ * struct xe_guc_relay - Data used by the VF-PF Relay Communication over GuC. */ struct xe_guc_relay { - /**@lock: protects all internal data. */ + /** @lock: protects all internal data. */ spinlock_t lock; /** @worker: dispatches incoming action messages. */ diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 42110e01b7d0..1c92b96e4c32 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -10,6 +10,7 @@ #include <linux/circ_buf.h> #include <linux/dma-fence-array.h> +#include <drm/drm_drv.h> #include <drm/drm_managed.h> #include "abi/guc_actions_abi.h" @@ -37,6 +38,7 @@ #include "xe_macros.h" #include "xe_map.h" #include "xe_mocs.h" +#include "xe_module.h" #include "xe_pm.h" #include "xe_ring_ops_types.h" #include "xe_sched_job.h" @@ -232,17 +234,9 @@ static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) static void guc_submit_sw_fini(struct drm_device *drm, void *arg) { struct xe_guc *guc = arg; - struct xe_device *xe = guc_to_xe(guc); struct xe_gt *gt = guc_to_gt(guc); - int ret; - - ret = wait_event_timeout(guc->submission_state.fini_wq, - xa_empty(&guc->submission_state.exec_queue_lookup), - HZ * 5); - drain_workqueue(xe->destroy_wq); - - xe_gt_assert(gt, ret); + xe_gt_assert(gt, xa_empty(&guc->submission_state.exec_queue_lookup)); xa_destroy(&guc->submission_state.exec_queue_lookup); } @@ -319,8 +313,6 @@ int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) xa_init(&guc->submission_state.exec_queue_lookup); - init_waitqueue_head(&guc->submission_state.fini_wq); - primelockdep(guc); guc->submission_state.initialized = true; @@ -397,46 +389,40 @@ void xe_guc_submit_disable(struct xe_guc *guc) guc->submission_state.enabled = false; } -static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count) +static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, + int count) { int i; - lockdep_assert_held(&guc->submission_state.lock); + mutex_lock(&guc->submission_state.lock); - for (i = 0; i < xa_count; ++i) - xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i); + for (i = 0; i < count; ++i) + xa_erase(&guc->submission_state.exec_queue_lookup, + q->guc->id + i); xe_guc_id_mgr_release_locked(&guc->submission_state.idm, q->guc->id, q->width); - if (xa_empty(&guc->submission_state.exec_queue_lookup)) - wake_up(&guc->submission_state.fini_wq); + mutex_unlock(&guc->submission_state.lock); } static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) { - int ret; - int i; - - /* - * Must use GFP_NOWAIT as this lock is in the dma fence signalling path, - * worse case user gets -ENOMEM on engine create and has to try again. - * - * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent - * failure. - */ - lockdep_assert_held(&guc->submission_state.lock); + int ret, i; + mutex_lock(&guc->submission_state.lock); ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, q->width); + mutex_unlock(&guc->submission_state.lock); if (ret < 0) return ret; q->guc->id = ret; + /* Reserve empty slots. */ for (i = 0; i < q->width; ++i) { - ret = xa_err(xa_store(&guc->submission_state.exec_queue_lookup, - q->guc->id + i, q, GFP_NOWAIT)); + ret = xa_insert(&guc->submission_state.exec_queue_lookup, + q->guc->id + i, NULL, GFP_KERNEL); if (ret) goto err_release; } @@ -449,11 +435,24 @@ err_release: return ret; } +static void publish_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) +{ + int i; + + lockdep_assert_held(&guc->submission_state.lock); + + for (i = 0; i < q->width; ++i) { + void *old; + + old = xa_store(&guc->submission_state.exec_queue_lookup, + q->guc->id + i, q, GFP_NOWAIT); + XE_WARN_ON(old || xa_is_err(old)); + } +} + static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) { - mutex_lock(&guc->submission_state.lock); __release_guc_id(guc, q, q->width); - mutex_unlock(&guc->submission_state.lock); } struct exec_queue_policy { @@ -841,10 +840,27 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1); } -static void __register_exec_queue_group(struct xe_guc *guc, - struct xe_exec_queue *q, +static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q) +{ +#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; + int len = 0; + + action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; + action[len++] = group->primary->guc->id; + + xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); +#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE + + xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); +} + +static void __register_exec_queue_group(struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) { + struct xe_guc *guc = exec_queue_to_guc(q); #define MAX_MULTI_QUEUE_REG_SIZE (8) u32 action[MAX_MULTI_QUEUE_REG_SIZE]; int len = 0; @@ -869,29 +885,6 @@ static void __register_exec_queue_group(struct xe_guc *guc, xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); } -static void xe_guc_exec_queue_group_add(struct xe_guc *guc, - struct xe_exec_queue *q) -{ -#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) - u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; - int len = 0; - - xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q)); - - action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; - action[len++] = q->multi_queue.group->primary->guc->id; - - xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); -#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE - - /* - * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a - * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response - * from guc. - */ - xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); -} - static void __register_mlrc_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) @@ -1017,7 +1010,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) set_exec_queue_registered(q); trace_xe_exec_queue_register(q); if (xe_exec_queue_is_multi_queue_primary(q)) - __register_exec_queue_group(guc, q, &info); + __register_exec_queue_group(q, &info); else if (xe_exec_queue_is_parallel(q)) __register_mlrc_exec_queue(guc, q, &info); else if (!xe_exec_queue_is_multi_queue_secondary(q)) @@ -1027,7 +1020,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) init_policies(guc, q); if (xe_exec_queue_is_multi_queue_secondary(q)) - xe_guc_exec_queue_group_add(guc, q); + guc_exec_queue_send_cgp_sync(q); } static u32 wq_space_until_wrap(struct xe_exec_queue *q) @@ -1159,7 +1152,7 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) if (exec_queue_suspended(q)) return; - if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { + if (!exec_queue_enabled(q)) { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; action[len++] = q->guc->id; action[len++] = GUC_CONTEXT_ENABLE; @@ -1208,10 +1201,8 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); - if (exec_queue_killed_or_banned_or_wedged(primary)) { - killed_or_banned_or_wedged = true; + if (exec_queue_killed_or_banned_or_wedged(primary)) goto run_job_out; - } if (!exec_queue_registered(primary)) register_exec_queue(primary, GUC_CONTEXT_NORMAL); @@ -1491,7 +1482,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) struct xe_device *xe = guc_to_xe(guc); int err = -ETIME; pid_t pid = -1; - bool wedged = false, skip_timeout_check; + bool wedged = false, wedge_device = false, skip_timeout_check; xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); @@ -1636,7 +1627,7 @@ trigger_reset: } if (q->flags & EXEC_QUEUE_FLAG_KERNEL) { xe_gt_WARN(q->gt, true, "Kernel-submitted job timed out\n"); - xe_device_declare_wedged(gt_to_xe(q->gt)); + wedge_device = true; } } else if (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)) { xe_gt_WARN(q->gt, true, "VM job timed out on non-killed execqueue\n"); @@ -1656,6 +1647,9 @@ trigger_reset: xe_guc_exec_queue_trigger_cleanup(q); } + if (wedge_device) + xe_device_declare_wedged(gt_to_xe(q->gt)); + /* * We want the job added back to the pending list so it gets freed; this * is what DRM_GPU_SCHED_STAT_NO_HANG does. @@ -1680,6 +1674,7 @@ static void guc_exec_queue_fini(struct xe_exec_queue *q) { struct xe_guc_exec_queue *ge = q->guc; struct xe_guc *guc = exec_queue_to_guc(q); + struct drm_device *drm = &guc_to_xe(guc)->drm; if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue_group *group = q->multi_queue.group; @@ -1698,36 +1693,52 @@ static void guc_exec_queue_fini(struct xe_exec_queue *q) * (timeline name). */ kfree_rcu(ge, rcu); + + drm_dev_put(drm); } -static void __guc_exec_queue_destroy_async(struct work_struct *w) +static void guc_exec_queue_do_destroy(struct xe_exec_queue *q) { - struct xe_guc_exec_queue *ge = - container_of(w, struct xe_guc_exec_queue, destroy_async); - struct xe_exec_queue *q = ge->q; + struct xe_guc_exec_queue *ge = q->guc; struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_device *xe = guc_to_xe(guc); + struct drm_device *drm = &xe->drm; + + /* + * guc_exec_queue_fini() drops the queue's drm_device ref. + * Keep the device alive until the PM-runtime guard unwinds. + */ + drm_dev_get(drm); - guard(xe_pm_runtime)(guc_to_xe(guc)); - trace_xe_exec_queue_destroy(q); + scoped_guard(xe_pm_runtime, xe) { + trace_xe_exec_queue_destroy(q); - /* Confirm no work left behind accessing device structures */ - cancel_delayed_work_sync(&ge->sched.base.work_tdr); + /* Confirm no work left behind accessing device structures */ + cancel_delayed_work_sync(&ge->sched.base.work_tdr); - xe_exec_queue_fini(q); + xe_exec_queue_fini(q); + } + + drm_dev_put(drm); } -static void guc_exec_queue_destroy_async(struct xe_exec_queue *q) +static void __guc_exec_queue_destroy_async(struct work_struct *w) { - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_device *xe = guc_to_xe(guc); + struct xe_guc_exec_queue *ge = + container_of(w, struct xe_guc_exec_queue, destroy_async); + + guc_exec_queue_do_destroy(ge->q); +} +static void guc_exec_queue_destroy_async(struct xe_exec_queue *q) +{ INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async); /* We must block on kernel engines so slabs are empty on driver unload */ if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q)) - __guc_exec_queue_destroy_async(&q->guc->destroy_async); + guc_exec_queue_do_destroy(q); else - queue_work(xe->destroy_wq, &q->guc->destroy_async); + xe_destroy_wq_queue(&q->guc->destroy_async); } static void __guc_exec_queue_destroy(struct xe_guc *guc, struct xe_exec_queue *q) @@ -1864,21 +1875,8 @@ static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sche { struct xe_exec_queue *q = msg->private_data; - if (guc_exec_queue_allowed_to_change_state(q)) { -#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_exec_queue_group *group = q->multi_queue.group; - u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; - int len = 0; - - action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; - action[len++] = group->primary->guc->id; - - xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); -#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE - - xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); - } + if (guc_exec_queue_allowed_to_change_state(q)) + guc_exec_queue_send_cgp_sync(q); kfree(msg); } @@ -1935,6 +1933,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched; struct xe_guc *guc = exec_queue_to_guc(q); + struct drm_device *drm = &guc_to_xe(guc)->drm; struct workqueue_struct *submit_wq = NULL; struct xe_guc_exec_queue *ge; long timeout; @@ -1946,6 +1945,8 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) if (!ge) return -ENOMEM; + drm_dev_get(drm); + q->guc = ge; ge->q = q; init_rcu_head(&ge->rcu); @@ -1957,6 +1958,14 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(q->sched_props.job_timeout_ms); + err = alloc_guc_id(guc, q); + if (err) + goto err_free; + + xe_exec_queue_assign_name(q, q->guc->id); + + strscpy(ge->name, q->name, sizeof(ge->name)); + /* * Use primary queue's submit_wq for all secondary queues of a * multi queue group. This serialization avoids any locking around @@ -1971,30 +1980,23 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64, timeout, guc_to_gt(guc)->ordered_wq, NULL, - q->name, gt_to_xe(q->gt)->drm.dev); + ge->name, gt_to_xe(q->gt)->drm.dev); if (err) - goto err_free; + goto err_release_id; sched = &ge->sched; err = xe_sched_entity_init(&ge->entity, sched); if (err) goto err_sched; - mutex_lock(&guc->submission_state.lock); - - err = alloc_guc_id(guc, q); - if (err) - goto err_entity; - q->entity = &ge->entity; + mutex_lock(&guc->submission_state.lock); if (xe_guc_read_stopped(guc) || vf_recovery(guc)) xe_sched_stop(sched); - + publish_guc_id(guc, q); mutex_unlock(&guc->submission_state.lock); - xe_exec_queue_assign_name(q, q->guc->id); - /* * Maintain secondary queues of the multi queue group in a list * for handling dependencies across the queues in the group. @@ -2017,13 +2019,13 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) return 0; -err_entity: - mutex_unlock(&guc->submission_state.lock); - xe_sched_entity_fini(&ge->entity); err_sched: xe_sched_fini(&ge->sched); +err_release_id: + release_guc_id(guc, q); err_free: kfree(ge); + drm_dev_put(drm); return err; } @@ -2943,9 +2945,10 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) if (unlikely(!q)) return -EPROTO; - xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id, - atomic_read(&q->guc->state)); + if (!exec_queue_killed(q)) + xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x", + xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id, + atomic_read(&q->guc->state)); trace_xe_exec_queue_reset(q); diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h index 5ccc5f959bb3..7824f61b1290 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit_types.h +++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h @@ -138,7 +138,7 @@ struct xe_guc_submit_exec_queue_snapshot { u32 primary; /** @multi_queue.pos: Position of the exec queue within the multi queue group */ u8 pos; - /** @valid: The exec queue is part of a multi queue group */ + /** @multi_queue.valid: The exec queue is part of a multi queue group */ bool valid; } multi_queue; }; diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h index c7b9642b41ba..31a2acb63ac3 100644 --- a/drivers/gpu/drm/xe/xe_guc_types.h +++ b/drivers/gpu/drm/xe/xe_guc_types.h @@ -100,8 +100,6 @@ struct xe_guc { * even initialized - before that not even the lock is valid */ bool initialized; - /** @submission_state.fini_wq: submit fini wait queue */ - wait_queue_head_t fini_wq; } submission_state; /** @hwconfig: Hardware config state */ diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 6dd05fac6595..0b193c451a11 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -67,7 +67,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 0, .irq_offset = ilog2(INTR_BCS(0)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = BLT_RING_BASE, }, [XE_HW_ENGINE_BCS1] = { @@ -75,7 +75,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 1, .irq_offset = ilog2(INTR_BCS(1)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS1_RING_BASE, }, [XE_HW_ENGINE_BCS2] = { @@ -83,7 +83,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 2, .irq_offset = ilog2(INTR_BCS(2)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS2_RING_BASE, }, [XE_HW_ENGINE_BCS3] = { @@ -91,7 +91,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 3, .irq_offset = ilog2(INTR_BCS(3)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS3_RING_BASE, }, [XE_HW_ENGINE_BCS4] = { @@ -99,7 +99,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 4, .irq_offset = ilog2(INTR_BCS(4)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS4_RING_BASE, }, [XE_HW_ENGINE_BCS5] = { @@ -107,7 +107,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 5, .irq_offset = ilog2(INTR_BCS(5)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS5_RING_BASE, }, [XE_HW_ENGINE_BCS6] = { @@ -115,7 +115,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 6, .irq_offset = ilog2(INTR_BCS(6)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS6_RING_BASE, }, [XE_HW_ENGINE_BCS7] = { @@ -123,7 +123,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .irq_offset = ilog2(INTR_BCS(7)), .instance = 7, - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS7_RING_BASE, }, [XE_HW_ENGINE_BCS8] = { @@ -131,7 +131,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 8, .irq_offset = ilog2(INTR_BCS8), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS8_RING_BASE, }, @@ -283,27 +283,6 @@ static void hw_engine_fini(void *arg) } /** - * xe_hw_engine_mmio_write32() - Write engine register - * @hwe: engine - * @reg: register to write into - * @val: desired 32-bit value to write - * - * This function will write val into an engine specific register. - * Forcewake must be held by the caller. - * - */ -void xe_hw_engine_mmio_write32(struct xe_hw_engine *hwe, - struct xe_reg reg, u32 val) -{ - xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base)); - xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain); - - reg.addr += hwe->mmio_base; - - xe_mmio_write32(&hwe->gt->mmio, reg, val); -} - -/** * xe_hw_engine_mmio_read32() - Read engine register * @hwe: engine * @reg: register to read from @@ -325,24 +304,8 @@ u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg) void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe) { - u32 ccs_mask = - xe_hw_engine_mask_per_class(hwe->gt, XE_ENGINE_CLASS_COMPUTE); - u32 ring_mode = REG_MASKED_FIELD_ENABLE(GFX_DISABLE_LEGACY_MODE); - - if (hwe->class == XE_ENGINE_CLASS_COMPUTE && ccs_mask) - xe_mmio_write32(&hwe->gt->mmio, RCU_MODE, - REG_MASKED_FIELD_ENABLE(RCU_MODE_CCS_ENABLE)); - - xe_hw_engine_mmio_write32(hwe, RING_HWSTAM(0), ~0x0); - xe_hw_engine_mmio_write32(hwe, RING_HWS_PGA(0), - xe_bo_ggtt_addr(hwe->hwsp)); - - if (xe_device_has_msix(gt_to_xe(hwe->gt))) - ring_mode |= REG_MASKED_FIELD_ENABLE(GFX_MSIX_INTERRUPT_ENABLE); - xe_hw_engine_mmio_write32(hwe, RING_MODE(0), ring_mode); - xe_hw_engine_mmio_write32(hwe, RING_MI_MODE(0), - REG_MASKED_FIELD_DISABLE(STOP_RING)); - xe_hw_engine_mmio_read32(hwe, RING_MI_MODE(0)); + xe_mmio_write32(&hwe->gt->mmio, RING_HWS_PGA(hwe->mmio_base), + xe_bo_ggtt_addr(hwe->hwsp)); } static bool xe_hw_engine_match_fixed_cslice_mode(const struct xe_device *xe, @@ -374,8 +337,8 @@ static bool xe_rtp_cfeg_wmtp_disabled(const struct xe_device *xe, return xe_mmio_read32(&hwe->gt->mmio, XEHP_FUSE4) & CFEG_WMTP_DISABLE; } -void -xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) +static void +hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) { struct xe_gt *gt = hwe->gt; const u8 mocs_write_idx = gt->mocs.uc_index; @@ -383,14 +346,14 @@ xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) u32 blit_cctl_val = REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, mocs_write_idx) | REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, mocs_read_idx); struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); - const struct xe_rtp_entry_sr lrc_setup[] = { + const struct xe_rtp_table_sr lrc_setup = XE_RTP_TABLE_SR( /* * Some blitter commands do not have a field for MOCS, those * commands will use MOCS index pointed by BLIT_CCTL. * BLIT_CCTL registers are needed to be programmed to un-cached. */ { XE_RTP_NAME("BLIT_CCTL_default_MOCS"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274), ENGINE_CLASS(COPY)), XE_RTP_ACTIONS(FIELD_SET(BLIT_CCTL(0), BLIT_CCTL_DST_MOCS_MASK | @@ -406,10 +369,20 @@ xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) PREEMPT_GPGPU_THREAD_GROUP_LEVEL)), XE_RTP_ENTRY_FLAG(FOREACH_ENGINE) }, - }; + ); - xe_rtp_process_to_sr(&ctx, lrc_setup, ARRAY_SIZE(lrc_setup), - &hwe->reg_lrc, true); + xe_rtp_process_to_sr(&ctx, &lrc_setup, &hwe->reg_lrc, true); +} + +void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + + xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe); + xe_wa_process_lrc(hwe); + hw_engine_setup_default_lrc_state(hwe); + xe_tuning_process_lrc(hwe); } static void @@ -434,15 +407,25 @@ hw_engine_setup_default_state(struct xe_hw_engine *hwe) u32 ring_cmd_cctl_val = REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, mocs_write_idx) | REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, mocs_read_idx); struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); - const struct xe_rtp_entry_sr engine_entries[] = { + const struct xe_rtp_table_sr engine_sr = XE_RTP_TABLE_SR( { XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), XE_RTP_ACTIONS(FIELD_SET(RING_CMD_CCTL(0), CMD_CCTL_WRITE_OVERRIDE_MASK | CMD_CCTL_READ_OVERRIDE_MASK, ring_cmd_cctl_val, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, + { XE_RTP_NAME("Disable HW status page updates for interrupts"), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), + XE_RTP_ACTIONS(SET(RING_HWSTAM(0), ~0x0, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("Disable engine 'legacy' mode"), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), + XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_DISABLE_LEGACY_MODE, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, /* * To allow the GSC engine to go idle on MTL we need to enable * idle messaging and set the hysteresis value (we use 0xA=5us @@ -465,16 +448,25 @@ hw_engine_setup_default_state(struct xe_hw_engine *hwe) XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), CS_PRIORITY_MEM_READ, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, + { XE_RTP_NAME("Enable CCS Engine(s)"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(RCU_MODE, RCU_MODE_CCS_ENABLE)) + }, /* Use Fixed slice CCS mode */ { XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"), XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)), XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE)) }, - }; + { XE_RTP_NAME("Enable MSI-X interrupt support"), + XE_RTP_RULES(FUNC(xe_rtp_match_has_msix)), + XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_MSIX_INTERRUPT_ENABLE, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + ); - xe_rtp_process_to_sr(&ctx, engine_entries, ARRAY_SIZE(engine_entries), - &hwe->reg_sr, false); + xe_rtp_process_to_sr(&ctx, &engine_sr, &hwe->reg_sr, false); } static const struct engine_info *find_engine_info(enum xe_engine_class class, int instance) @@ -520,9 +512,14 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, hwe->class = info->class; hwe->instance = info->instance; hwe->mmio_base = info->mmio_base; - hwe->irq_offset = xe_device_has_msix(gt_to_xe(gt)) ? - get_msix_irq_offset(gt, info->class) : - info->irq_offset; + if (xe_device_has_msix(gt_to_xe(gt))) { + hwe->irq_offset = get_msix_irq_offset(gt, info->class); + hwe->irq_page = info->instance; + + } else { + hwe->irq_offset = info->irq_offset; + hwe->irq_page = 0; + } hwe->domain = info->domain; hwe->name = info->name; hwe->fence_irq = >->fence_irq[info->class]; @@ -575,6 +572,8 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, hw_engine_setup_default_state(hwe); xe_reg_sr_init(&hwe->reg_whitelist, hwe->name, gt_to_xe(gt)); + xe_reg_sr_init(&hwe->oa_whitelist, hwe->name, gt_to_xe(gt)); + xe_reg_sr_init(&hwe->oa_sr, hwe->name, gt_to_xe(gt)); xe_reg_whitelist_process_engine(hwe); } @@ -629,7 +628,7 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, hwe->exl_port = xe_execlist_port_create(xe, hwe); if (IS_ERR(hwe->exl_port)) { err = PTR_ERR(hwe->exl_port); - goto err_hwsp; + goto err_name; } } else { /* GSCCS has a special interrupt for reset */ @@ -649,8 +648,6 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, return devm_add_action_or_reset(xe->drm.dev, hw_engine_fini, hwe); -err_hwsp: - xe_bo_unpin_map_no_vm(hwe->hwsp); err_name: hwe->name = NULL; @@ -1026,6 +1023,9 @@ bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe) struct xe_gt *gt = hwe->gt; struct xe_device *xe = gt_to_xe(gt); + if (xe_device_is_admin_only(xe)) + return true; + if (hwe->class == XE_ENGINE_CLASS_OTHER) return true; diff --git a/drivers/gpu/drm/xe/xe_hw_engine.h b/drivers/gpu/drm/xe/xe_hw_engine.h index 6b5f9fa2a594..c3ee37f8cfc0 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.h +++ b/drivers/gpu/drm/xe/xe_hw_engine.h @@ -59,7 +59,7 @@ struct xe_hw_engine_snapshot * xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe, struct xe_exec_queue *q); void xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot *snapshot); void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p); -void xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe); +void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe); bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe); @@ -76,7 +76,6 @@ const char *xe_hw_engine_class_to_str(enum xe_engine_class class); u64 xe_hw_engine_read_timestamp(struct xe_hw_engine *hwe); enum xe_force_wake_domains xe_hw_engine_to_fw_domain(struct xe_hw_engine *hwe); -void xe_hw_engine_mmio_write32(struct xe_hw_engine *hwe, struct xe_reg reg, u32 val); u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg); #endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine_types.h b/drivers/gpu/drm/xe/xe_hw_engine_types.h index e4191a7a2c31..84c097da9b6f 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_types.h +++ b/drivers/gpu/drm/xe/xe_hw_engine_types.h @@ -79,7 +79,7 @@ struct xe_hw_engine_class_intf { * @defaults: default scheduling properties */ struct { - /** @sched_props.set_job_timeout: Set job timeout in ms for engine */ + /** @sched_props.job_timeout_ms: Set job timeout in ms for engine */ u32 job_timeout_ms; /** @sched_props.job_timeout_min: Min job timeout in ms for engine */ u32 job_timeout_min; @@ -118,6 +118,8 @@ struct xe_hw_engine { u16 logical_instance; /** @irq_offset: IRQ offset of this hw engine */ u16 irq_offset; + /** @irq_page: MEMIRQ page used by this HW engine */ + u16 irq_page; /** @mmio_base: MMIO base address of this hw engine*/ u32 mmio_base; /** @@ -129,6 +131,14 @@ struct xe_hw_engine { */ struct xe_reg_sr reg_whitelist; /** + * @oa_whitelist: oa registers to be whitelisted + */ + struct xe_reg_sr oa_whitelist; + /** + * @oa_sr: oa nonpriv whitelist registers, changed on oa stream open/close + */ + struct xe_reg_sr oa_sr; + /** * @reg_lrc: LRC workaround registers */ struct xe_reg_sr reg_lrc; diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c index e869bc3948d9..4b72959b2276 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.c +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -36,11 +36,6 @@ static const char * const hec_uncorrected_fw_errors[] = { "Data Corruption" }; -static const unsigned long xe_hw_error_map[] = { - [XE_GT_ERROR] = DRM_XE_RAS_ERR_COMP_CORE_COMPUTE, - [XE_SOC_ERROR] = DRM_XE_RAS_ERR_COMP_SOC_INTERNAL, -}; - enum gt_vector_regs { ERR_STAT_GT_VECTOR0 = 0, ERR_STAT_GT_VECTOR1, @@ -65,6 +60,18 @@ static enum drm_xe_ras_error_severity hw_err_to_severity(const enum hardware_err return DRM_XE_RAS_ERR_SEV_UNCORRECTABLE; } +static inline u32 err_src_to_id(u32 err_bit) +{ + switch (err_bit) { + case XE_GT_ERROR: + return DRM_XE_RAS_ERR_COMP_CORE_COMPUTE; + case XE_SOC_ERROR: + return DRM_XE_RAS_ERR_COMP_SOC_INTERNAL; + default: + return 0; + } +} + static const char * const pvc_master_global_err_reg[] = { [0 ... 1] = "Undefined", [2] = "HBM SS0: Channel0", @@ -169,11 +176,8 @@ static void csc_hw_error_work(struct work_struct *work) { struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work); struct xe_device *xe = tile_to_xe(tile); - int ret; - ret = xe_survivability_mode_runtime_enable(xe); - if (ret) - drm_err(&xe->drm, "Failed to enable runtime survivability mode\n"); + xe_survivability_mode_runtime_enable(xe); } static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err) @@ -459,14 +463,8 @@ static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_er const char *name; u32 error_id; - /* Check error bit is within bounds */ - if (err_bit >= ARRAY_SIZE(xe_hw_error_map)) - break; - - error_id = xe_hw_error_map[err_bit]; - - /* Check error component is within max */ - if (!error_id || error_id >= DRM_XE_RAS_ERR_COMP_MAX) + error_id = err_src_to_id(err_bit); + if (!error_id) continue; name = info[error_id].name; diff --git a/drivers/gpu/drm/xe/xe_hw_error.h b/drivers/gpu/drm/xe/xe_hw_error.h index d86e28c5180c..5e3a11424108 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.h +++ b/drivers/gpu/drm/xe/xe_hw_error.h @@ -2,8 +2,8 @@ /* * Copyright © 2025 Intel Corporation */ -#ifndef XE_HW_ERROR_H_ -#define XE_HW_ERROR_H_ +#ifndef _XE_HW_ERROR_H_ +#define _XE_HW_ERROR_H_ #include <linux/types.h> diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index 92e423a339f1..de3f2aeffc3f 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -180,6 +180,18 @@ struct xe_hwmon { struct xe_hwmon_thermal_info temp; }; +static inline int prepare_power_limit_param2(const struct xe_hwmon *hwmon) +{ + if (hwmon->boot_power_limit_read) { + if (hwmon->xe->info.platform >= XE_CRESCENTISLAND) + return READ_PL_ACCEPTED; + else + return READ_PL_FROM_PCODE; + } else { + return READ_PL_FROM_FW; + } +} + static int xe_hwmon_pcode_read_power_limit(const struct xe_hwmon *hwmon, u32 attr, int channel, u32 *uval) { @@ -191,9 +203,7 @@ static int xe_hwmon_pcode_read_power_limit(const struct xe_hwmon *hwmon, u32 att (channel == CHANNEL_CARD) ? READ_PSYSGPU_POWER_LIMIT : READ_PACKAGE_POWER_LIMIT, - hwmon->boot_power_limit_read ? - READ_PL_FROM_PCODE : READ_PL_FROM_FW), - &val0, &val1); + prepare_power_limit_param2(hwmon)), &val0, &val1); if (ret) { drm_dbg(&hwmon->xe->drm, "read failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n", @@ -226,10 +236,7 @@ static int xe_hwmon_pcode_rmw_power_limit(const struct xe_hwmon *hwmon, u32 attr (channel == CHANNEL_CARD) ? READ_PSYSGPU_POWER_LIMIT : READ_PACKAGE_POWER_LIMIT, - hwmon->boot_power_limit_read ? - READ_PL_FROM_PCODE : READ_PL_FROM_FW), - &val0, &val1); - + prepare_power_limit_param2(hwmon)), &val0, &val1); if (ret) drm_dbg(&hwmon->xe->drm, "read failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n", channel, val0, val1, ret); @@ -296,7 +303,12 @@ static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg return GT_PERF_STATUS; break; case REG_PKG_ENERGY_STATUS: - if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) { + if (xe->info.platform == XE_CRESCENTISLAND) { + if (channel == CHANNEL_CARD) + return CRI_PLATFORM_ENERGY_STATUS; + else if (channel == CHANNEL_PKG) + return CRI_PACKAGE_ENERGY_STATUS; + } else if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) { return PVC_GT0_PLATFORM_ENERGY_STATUS; } else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG)) { return PCU_CR_PACKAGE_ENERGY_STATUS; diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c index 7560a45f7f64..9e49e2241da4 100644 --- a/drivers/gpu/drm/xe/xe_irq.c +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -24,6 +24,7 @@ #include "xe_mmio.h" #include "xe_pxp.h" #include "xe_sriov.h" +#include "xe_sysctrl.h" #include "xe_tile.h" /* @@ -525,6 +526,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) xe_heci_csc_irq_handler(xe, master_ctl); xe_display_irq_handler(xe, master_ctl); xe_i2c_irq_handler(xe, master_ctl); + xe_sysctrl_irq_handler(xe, master_ctl); xe_mert_irq_handler(xe, master_ctl); gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); } diff --git a/drivers/gpu/drm/xe/xe_late_bind_fw_types.h b/drivers/gpu/drm/xe/xe_late_bind_fw_types.h index 7fdb24e810b3..ee5efe60774e 100644 --- a/drivers/gpu/drm/xe/xe_late_bind_fw_types.h +++ b/drivers/gpu/drm/xe/xe_late_bind_fw_types.h @@ -10,7 +10,8 @@ #include <linux/mutex.h> #include <linux/types.h> #include <linux/workqueue.h> -#include "xe_uc_fw_abi.h" + +#include "abi/uc_fw_abi.h" #define XE_LB_MAX_PAYLOAD_SIZE SZ_4K diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 4af9f0d7c6f3..a4292a11391d 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -21,8 +21,10 @@ #include "xe_configfs.h" #include "xe_device.h" #include "xe_drm_client.h" +#include "xe_exec_queue.h" #include "xe_exec_queue_types.h" #include "xe_gt.h" +#include "xe_gt_clock.h" #include "xe_gt_printk.h" #include "xe_hw_fence.h" #include "xe_map.h" @@ -682,25 +684,6 @@ static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe) } } -static int lrc_ring_mi_mode(struct xe_hw_engine *hwe) -{ - struct xe_device *xe = gt_to_xe(hwe->gt); - - if (GRAPHICS_VERx100(xe) >= 1250) - return 0x70; - else - return 0x60; -} - -static void reset_stop_ring(u32 *regs, struct xe_hw_engine *hwe) -{ - int x; - - x = lrc_ring_mi_mode(hwe); - regs[x + 1] &= ~STOP_RING; - regs[x + 1] |= STOP_RING << 16; -} - static inline bool xe_lrc_has_indirect_ring_state(struct xe_lrc *lrc) { return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE; @@ -795,6 +778,16 @@ static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc) return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32); } +static u32 __xe_lrc_queue_timestamp_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP * sizeof(u32); +} + +static u32 __xe_lrc_queue_timestamp_udw_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP_UDW * sizeof(u32); +} + static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc) { u32 offset = xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - @@ -844,6 +837,8 @@ DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo) DECL_MAP_ADDR_HELPERS(parallel, lrc->bo) DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo) DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo) +DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo) +DECL_MAP_ADDR_HELPERS(queue_timestamp_udw, lrc->bo) #undef DECL_MAP_ADDR_HELPERS @@ -893,6 +888,29 @@ static u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc) } /** + * xe_lrc_queue_timestamp() - Read queue timestamp value + * @lrc: Pointer to the lrc. + * + * Returns: queue timestamp value + */ +static u64 xe_lrc_queue_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + u32 ldw, udw = 0; + + xe_assert(xe, xe_lrc_is_multi_queue(lrc)); + + map = __xe_lrc_queue_timestamp_map(lrc); + ldw = xe_map_read32(xe, &map); + + map = __xe_lrc_queue_timestamp_udw_map(lrc); + udw = xe_map_read32(xe, &map); + + return (u64)udw << 32 | ldw; +} + +/** * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address * @lrc: Pointer to the lrc. * @@ -987,7 +1005,6 @@ static void *empty_lrc_data(struct xe_hw_engine *hwe) set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe); set_context_control(regs, hwe); set_memory_based_intr(regs, hwe); - reset_stop_ring(regs, hwe); if (xe_gt_has_indirect_ring_state(gt)) { regs = data + xe_gt_lrc_size(gt, hwe->class) - LRC_INDIRECT_RING_STATE_SIZE; @@ -1557,6 +1574,18 @@ static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct if (lrc_to_xe(lrc)->info.has_64bit_timestamp) xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0); + /* + * Note: It's possible that this LRC may belong to an exec_queue that is + * not part of a multi-queue group. That said, it doesn't hurt to set + * this field anyways since any class that supports multi-queue will + * have these LRC fields defined. + */ + if (xe_gt_supports_multi_queue(gt, hwe->class)) { + lrc->queue_timestamp = 0; + xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP, 0); + xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP_UDW, 0); + } + if (xe->info.has_asid && vm) xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid); @@ -2482,7 +2511,17 @@ struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) snapshot->replay_offset = 0; snapshot->replay_size = lrc->replay_size; snapshot->lrc_snapshot = NULL; - snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc)); + snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); + snapshot->ctx_timestamp_ms = + xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc)); + if (xe_lrc_is_multi_queue(lrc)) { + snapshot->queue_timestamp = xe_lrc_queue_timestamp(lrc); + snapshot->queue_timestamp_ms = + xe_gt_clock_interval_to_ms(lrc->gt, snapshot->queue_timestamp); + } else { + snapshot->queue_timestamp = 0; + snapshot->queue_timestamp_ms = 0; + } snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); return snapshot; } @@ -2535,7 +2574,10 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); - drm_printf(p, "\tTimestamp: 0x%08x\n", snapshot->ctx_timestamp); + drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp); + drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms); + drm_printf(p, "\tQueue Timestamp: 0x%016llx\n", snapshot->queue_timestamp); + drm_printf(p, "\tQueue Timestamp ms: %llu\n", snapshot->queue_timestamp_ms); drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); if (!snapshot->lrc_snapshot) @@ -2576,17 +2618,27 @@ void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot) kfree(snapshot); } -static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) +static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id) { u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id); u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id); + struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false); + + if (xe_gt_WARN_ONCE(gt, !hwe || xe_hw_engine_is_reserved(hwe), + "Unexpected engine class:instance %d:%d for utilization\n", + class, instance)) + return NULL; + + return hwe; +} + +static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) +{ struct xe_hw_engine *hwe; u64 val; - hwe = xe_gt_hw_engine(lrc->gt, class, instance, false); - if (xe_gt_WARN_ONCE(lrc->gt, !hwe || xe_hw_engine_is_reserved(hwe), - "Unexpected engine class:instance %d:%d for context utilization\n", - class, instance)) + hwe = engine_id_to_hwe(lrc->gt, engine_id); + if (!hwe) return -1; if (lrc_to_xe(lrc)->info.has_64bit_timestamp) @@ -2601,66 +2653,136 @@ static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) return 0; } -/** - * xe_lrc_timestamp() - Current ctx timestamp - * @lrc: Pointer to the lrc. - * - * Return latest ctx timestamp. With support for active contexts, the - * calculation may be slightly racy, so follow a read-again logic to ensure that - * the context is still active before returning the right timestamp. - * - * Returns: New ctx timestamp value - */ -u64 xe_lrc_timestamp(struct xe_lrc *lrc) +static u64 get_queue_timestamp(struct xe_hw_engine *hwe) { - u64 lrc_ts, reg_ts, new_ts = lrc->ctx_timestamp; - u32 engine_id; + return xe_mmio_read64_2x32(&hwe->gt->mmio, + RING_QUEUE_TIMESTAMP(hwe->mmio_base)); +} - lrc_ts = xe_lrc_ctx_timestamp(lrc); - /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ - if (IS_SRIOV_VF(lrc_to_xe(lrc))) { - new_ts = lrc_ts; - goto done; - } +static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe) +{ + u32 val = xe_mmio_read32(&hwe->gt->mmio, + RING_CSMQDEBUG(hwe->mmio_base)); - if (lrc_ts == CONTEXT_ACTIVE) { - engine_id = xe_lrc_engine_id(lrc); - if (!get_ctx_timestamp(lrc, engine_id, ®_ts)) - new_ts = reg_ts; + return REG_FIELD_GET(CURRENT_ACTIVE_QUEUE_ID_MASK, val); +} - /* read lrc again to ensure context is still active */ - lrc_ts = xe_lrc_ctx_timestamp(lrc); - } +static bool context_active(struct xe_lrc *lrc) +{ + return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE; +} + +static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct xe_lrc *primary_lrc = lrc->multi_queue.primary_lrc; + struct xe_hw_engine *hwe; + u64 reg_queue_ts = lrc->queue_timestamp; + + if (IS_SRIOV_VF(xe)) + return xe_lrc_queue_timestamp(lrc); + + xe_assert(xe, primary_lrc); + + /* WA BB populates CONTEXT_ACTIVE cookie for primary context only */ + if (!context_active(primary_lrc)) + return xe_lrc_queue_timestamp(lrc); + + /* WA BB populates engine id in PPHWSP of primary context only */ + hwe = engine_id_to_hwe(primary_lrc->gt, xe_lrc_engine_id(primary_lrc)); + if (!hwe) + return xe_lrc_queue_timestamp(lrc); + + if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos) + return xe_lrc_queue_timestamp(lrc); + + /* queue is active, so store the queue timestamp register */ + reg_queue_ts = get_queue_timestamp(hwe); + + /* double check queue and primary queue are both still active */ + if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos || + !context_active(primary_lrc)) + return xe_lrc_queue_timestamp(lrc); + + return reg_queue_ts; +} + +static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + *old_ts = lrc->queue_timestamp; + lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc); + + trace_xe_lrc_update_queue_timestamp(lrc, *old_ts); + + return lrc->queue_timestamp; +} + +static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc) +{ + u64 reg_ts, new_ts = lrc->ctx_timestamp; + + /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ + if (IS_SRIOV_VF(lrc_to_xe(lrc))) + return xe_lrc_ctx_timestamp(lrc); + + if (context_active(lrc) && + !get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts)) + new_ts = reg_ts; /* - * If context switched out, just use the lrc_ts. Note that this needs to - * be a separate if condition. + * If context swicthed out while we were here, just return the latest + * LRC CTX TIMESTAMP value. */ - if (lrc_ts != CONTEXT_ACTIVE) - new_ts = lrc_ts; + if (!context_active(lrc)) + return xe_lrc_ctx_timestamp(lrc); -done: return new_ts; } +static u64 xe_lrc_update_context_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + *old_ts = lrc->ctx_timestamp; + lrc->ctx_timestamp = xe_lrc_context_timestamp(lrc); + + trace_xe_lrc_update_timestamp(lrc, *old_ts); + + return lrc->ctx_timestamp; +} + +/** + * xe_lrc_timestamp() - Current lrc timestamp + * @lrc: Pointer to the lrc. + * + * Return latest lrc timestamp. With support for active contexts/queues, the + * calculation may be slightly racy, so follow a read-again logic to ensure that + * the context/queue is still active before returning the right timestamp. + * + * Returns: New lrc timestamp value + */ +u64 xe_lrc_timestamp(struct xe_lrc *lrc) +{ + if (xe_lrc_is_multi_queue(lrc)) + return xe_lrc_multi_queue_timestamp(lrc); + else + return xe_lrc_context_timestamp(lrc); +} + /** - * xe_lrc_update_timestamp() - Update ctx timestamp + * xe_lrc_update_timestamp() - Update lrc timestamp * @lrc: Pointer to the lrc. * @old_ts: Old timestamp value * - * Populate @old_ts current saved ctx timestamp, read new ctx timestamp and + * Populate @old_ts with current saved lrc timestamp, read new lrc timestamp and * update saved value. * - * Returns: New ctx timestamp value + * Returns: New lrc timestamp value */ u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts) { - *old_ts = lrc->ctx_timestamp; - lrc->ctx_timestamp = xe_lrc_timestamp(lrc); - - trace_xe_lrc_update_timestamp(lrc, *old_ts); - - return lrc->ctx_timestamp; + if (xe_lrc_is_multi_queue(lrc)) + return xe_lrc_update_multi_queue_timestamp(lrc, old_ts); + else + return xe_lrc_update_context_timestamp(lrc, old_ts); } /** diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index 5440663183f6..0a3a611391ee 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -37,7 +37,10 @@ struct xe_lrc_snapshot { } tail; u32 start_seqno; u32 seqno; - u32 ctx_timestamp; + u64 ctx_timestamp; + u64 ctx_timestamp_ms; + u64 queue_timestamp; + u64 queue_timestamp_ms; u32 ctx_job_timestamp; }; @@ -90,6 +93,11 @@ static inline size_t xe_lrc_ring_size(void) return SZ_16K; } +static inline bool xe_lrc_is_multi_queue(struct xe_lrc *lrc) +{ + return lrc->multi_queue.primary_lrc; +} + size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class); size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class); u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc); diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h index 5a718f759ed6..53ef48feebfc 100644 --- a/drivers/gpu/drm/xe/xe_lrc_types.h +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -63,6 +63,17 @@ struct xe_lrc { /** @ctx_timestamp: readout value of CTX_TIMESTAMP on last update */ u64 ctx_timestamp; + + /** @queue_timestamp: value of QUEUE_TIMESTAMP on last update */ + u64 queue_timestamp; + + /** @multi_queue: Multi queue LRC related information */ + struct { + /** @multi_queue.primary_lrc: Primary lrc of this multi-queue group*/ + struct xe_lrc *primary_lrc; + /** @multi_queue.pos: Position of LRC within the multi-queue group */ + u8 pos; + } multi_queue; }; struct xe_lrc_snapshot; diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c index 579af47edc61..9dfe965cb46e 100644 --- a/drivers/gpu/drm/xe/xe_memirq.c +++ b/drivers/gpu/drm/xe/xe_memirq.c @@ -152,6 +152,16 @@ static const char *guc_name(struct xe_guc *guc) * */ +/* ISR */ +#define XE_MEMIRQ_STATUS_OFFSET(inst) ((inst) * SZ_4K + 0x0) +/* IIR */ +#define XE_MEMIRQ_SOURCE_OFFSET(inst) ((inst) * SZ_4K + 0x400) +/* IMR */ +#define XE_MEMIRQ_ENABLE_OFFSET 0x440 + +/* engine ISR vector offset */ +#define XE_MEMIRQ_VECTOR_OFFSET(page, source) (XE_MEMIRQ_STATUS_OFFSET(page) + (source) * SZ_16) + static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq) { /* @@ -162,18 +172,35 @@ static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq) return xe_device_has_msix(memirq_to_xe(memirq)); } +static unsigned int hwe_max_count(struct xe_tile *tile) +{ + unsigned int max_instance = 0; + unsigned int gtid, hweid; + struct xe_hw_engine *hwe; + struct xe_gt *gt; + + for_each_gt_on_tile(gt, tile, gtid) + for_each_hw_engine(hwe, gt, hweid) + max_instance = max(max_instance, hwe->instance); + + return max_instance + 1; +} + static int memirq_alloc_pages(struct xe_memirq *memirq) { struct xe_device *xe = memirq_to_xe(memirq); struct xe_tile *tile = memirq_to_tile(memirq); - size_t bo_size = hw_reports_to_instance_zero(memirq) ? - XE_HW_ENGINE_MAX_INSTANCE * SZ_4K : SZ_4K; + unsigned int num_pages; struct xe_bo *bo; + size_t bo_size; int err; BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET(0), SZ_64)); BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET(0), SZ_4K)); + num_pages = hw_reports_to_instance_zero(memirq) ? hwe_max_count(tile) : 1; + bo_size = num_pages * SZ_4K; + bo = xe_managed_bo_create_pin_map(xe, tile, bo_size, XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT | @@ -192,16 +219,15 @@ static int memirq_alloc_pages(struct xe_memirq *memirq) memirq->bo = bo; memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET(0)); - memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET(0)); - memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET); + memirq->num_pages = num_pages; memirq_assert(memirq, !memirq->source.is_iomem); - memirq_assert(memirq, !memirq->status.is_iomem); - memirq_assert(memirq, !memirq->mask.is_iomem); - memirq_debug(memirq, "page offsets: bo %#x bo_size %zu source %#x status %#x\n", - xe_bo_ggtt_addr(bo), bo_size, XE_MEMIRQ_SOURCE_OFFSET(0), - XE_MEMIRQ_STATUS_OFFSET(0)); + memirq_debug(memirq, "pages: count %u size %zu\n", num_pages, bo_size); + memirq_debug(memirq, "page0: source %#x status %#x mask %#x\n", + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_SOURCE_OFFSET(0), + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_STATUS_OFFSET(0), + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_ENABLE_OFFSET); return 0; @@ -212,7 +238,12 @@ out: static void memirq_set_enable(struct xe_memirq *memirq, bool enable) { - iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0); + /* + * We only care about the GT_MI_USER_INTERRUPT from the engines and + * the GuC does not look at the ENABLE mask at all. + */ + iosys_map_wr(&memirq->bo->vmap, XE_MEMIRQ_ENABLE_OFFSET, u32, + enable ? GT_MI_USER_INTERRUPT : 0); memirq->enabled = enable; } @@ -249,15 +280,6 @@ int xe_memirq_init(struct xe_memirq *memirq) return 0; } -static u32 __memirq_source_page(struct xe_memirq *memirq, u16 instance) -{ - memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE); - memirq_assert(memirq, memirq->bo); - - instance = hw_reports_to_instance_zero(memirq) ? instance : 0; - return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(instance); -} - /** * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_. * @memirq: the &xe_memirq to query @@ -272,16 +294,7 @@ u32 xe_memirq_source_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe) { memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); - return __memirq_source_page(memirq, hwe->instance); -} - -static u32 __memirq_status_page(struct xe_memirq *memirq, u16 instance) -{ - memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE); - memirq_assert(memirq, memirq->bo); - - instance = hw_reports_to_instance_zero(memirq) ? instance : 0; - return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(instance); + return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(hwe->irq_page); } /** @@ -298,7 +311,7 @@ u32 xe_memirq_status_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe) { memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); - return __memirq_status_page(memirq, hwe->instance); + return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(hwe->irq_page); } /** @@ -335,13 +348,14 @@ int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc) { bool is_media = xe_gt_is_media_type(guc_to_gt(guc)); u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC); - u32 source, status; + u64 source, status; int err; memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); - source = __memirq_source_page(memirq, 0) + offset; - status = __memirq_status_page(memirq, 0) + offset * SZ_16; + /* GuC expects exact locations, it doesn't add anything on its own */ + source = xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(0) + offset; + status = xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_VECTOR_OFFSET(0, offset); err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY, source); @@ -485,16 +499,16 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat */ void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe) { - u16 offset = hwe->irq_offset; - u16 instance = hw_reports_to_instance_zero(memirq) ? hwe->instance : 0; - struct iosys_map src_offset = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, - XE_MEMIRQ_SOURCE_OFFSET(instance)); + struct iosys_map source = + IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_SOURCE_OFFSET(hwe->irq_page)); - if (memirq_received(memirq, &src_offset, offset, "SRC")) { - struct iosys_map status_offset = + if (memirq_received(memirq, &source, hwe->irq_offset, "SRC")) { + struct iosys_map status = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, - XE_MEMIRQ_STATUS_OFFSET(instance) + offset * SZ_16); - memirq_dispatch_engine(memirq, &status_offset, hwe); + XE_MEMIRQ_VECTOR_OFFSET(hwe->irq_page, + hwe->irq_offset)); + memirq_dispatch_engine(memirq, &status, hwe); } } @@ -509,12 +523,25 @@ bool xe_memirq_guc_sw_int_0_irq_pending(struct xe_memirq *memirq, struct xe_guc { struct xe_gt *gt = guc_to_gt(guc); u32 offset = xe_gt_is_media_type(gt) ? ilog2(INTR_MGUC) : ilog2(INTR_GUC); - struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&memirq->status, offset * SZ_16); + struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, offset)); return memirq_received_noclear(memirq, &map, ilog2(GUC_INTR_SW_INT_0), guc_name(guc)); } +static void memirq_dump_source_pages(struct xe_memirq *memirq) +{ + memirq_assert(memirq, !memirq->bo->vmap.is_iomem); + + for (int n = 0; n < memirq->num_pages; n++) { + memirq_debug(memirq, "SOURCE %*ph\n", 32, + memirq->bo->vmap.vaddr + XE_MEMIRQ_SOURCE_OFFSET(n)); + memirq_debug(memirq, "SOURCE %*ph\n", 32, + memirq->bo->vmap.vaddr + XE_MEMIRQ_SOURCE_OFFSET(n) + 32); + } +} + /** * xe_memirq_handler - The `Memory Based Interrupts`_ Handler. * @memirq: the &xe_memirq @@ -534,9 +561,8 @@ void xe_memirq_handler(struct xe_memirq *memirq) if (!memirq->bo) return; - memirq_assert(memirq, !memirq->source.is_iomem); - memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr); - memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32); + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ)) + memirq_dump_source_pages(memirq); for_each_gt(gt, xe, gtid) { if (gt->tile != tile) @@ -549,7 +575,8 @@ void xe_memirq_handler(struct xe_memirq *memirq) /* GuC and media GuC (if present) must be checked separately */ if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) { - map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16); + map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, ilog2(INTR_GUC))); memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc); } @@ -557,7 +584,8 @@ void xe_memirq_handler(struct xe_memirq *memirq) return; if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) { - map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16); + map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, ilog2(INTR_MGUC))); memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc); } } diff --git a/drivers/gpu/drm/xe/xe_memirq_types.h b/drivers/gpu/drm/xe/xe_memirq_types.h index 9d0f6c1cdb9d..c2b0b33d55cd 100644 --- a/drivers/gpu/drm/xe/xe_memirq_types.h +++ b/drivers/gpu/drm/xe/xe_memirq_types.h @@ -10,27 +10,18 @@ struct xe_bo; -/* ISR */ -#define XE_MEMIRQ_STATUS_OFFSET(inst) ((inst) * SZ_4K + 0x0) -/* IIR */ -#define XE_MEMIRQ_SOURCE_OFFSET(inst) ((inst) * SZ_4K + 0x400) -/* IMR */ -#define XE_MEMIRQ_ENABLE_OFFSET 0x440 - /** * struct xe_memirq - Data used by the `Memory Based Interrupts`_. * * @bo: buffer object with `Memory Based Interrupts Page Layout`_. + * @num_pages: number of per-instance source/status pages. * @source: iosys pointer to `Interrupt Source Report Page`_. - * @status: iosys pointer to `Interrupt Status Report Page`_. - * @mask: iosys pointer to Interrupt Enable Mask. * @enabled: internal flag used to control processing of the interrupts. */ struct xe_memirq { struct xe_bo *bo; + unsigned int num_pages; struct iosys_map source; - struct iosys_map status; - struct iosys_map mask; bool enabled; }; diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index a22413f892a0..7d28290e7d1c 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -31,6 +31,7 @@ #include "xe_map.h" #include "xe_mem_pool.h" #include "xe_mocs.h" +#include "xe_pat.h" #include "xe_printk.h" #include "xe_pt.h" #include "xe_res_cursor.h" @@ -217,7 +218,7 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, struct xe_vm *vm, u32 *ofs) { struct xe_device *xe = tile_to_xe(tile); - u16 pat_index = xe->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); u8 id = tile->id; u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; #define VRAM_IDENTITY_MAP_COUNT 2 @@ -337,7 +338,7 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, * if flat ccs is enabled. */ if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) { - u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION]; + u16 comp_pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION); u64 vram_offset = IDENTITY_OFFSET + DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G); u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE; @@ -637,10 +638,10 @@ static void emit_pte(struct xe_migrate *m, /* Indirect access needs compression enabled uncached PAT index */ if (GRAPHICS_VERx100(xe) >= 2000) - pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] : - xe->pat.idx[XE_CACHE_WB]; + pat_index = is_comp_pte ? xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION) : + xe_cache_pat_idx(xe, XE_CACHE_WB); else - pat_index = xe->pat.idx[XE_CACHE_WB]; + pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); @@ -727,7 +728,22 @@ static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, bb->len = cs - bb->cs; } -#define EMIT_COPY_DW 10 +static u32 blt_fast_copy_cmd_len(struct xe_device *xe) +{ + return 10; +} + +static u32 blt_mem_copy_cmd_len(struct xe_device *xe) +{ + return 10; +} + +static u32 emit_copy_cmd_len(struct xe_device *xe) +{ + return (xe->info.has_mem_copy_instr) ? blt_mem_copy_cmd_len(xe) : + blt_fast_copy_cmd_len(xe); +} + static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u64 dst_ofs, unsigned int size, unsigned int pitch) @@ -735,6 +751,7 @@ static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, struct xe_device *xe = gt_to_xe(gt); u32 mocs = 0; u32 tile_y = 0; + u32 len; xe_gt_assert(gt, !(pitch & 3)); xe_gt_assert(gt, size / pitch <= S16_MAX); @@ -747,7 +764,8 @@ static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, if (GRAPHICS_VERx100(xe) >= 1250) tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; - bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); + len = blt_fast_copy_cmd_len(xe); + bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (len - 2); bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; bb->cs[bb->len++] = 0; bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; @@ -764,6 +782,7 @@ static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u64 dst_ofs, unsigned int size, unsigned int pitch) { u32 mode, copy_type, width; + u32 len; xe_gt_assert(gt, IS_ALIGNED(size, pitch)); xe_gt_assert(gt, pitch <= U16_MAX); @@ -789,7 +808,9 @@ static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, xe_gt_assert(gt, width <= U16_MAX); - bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type; + len = blt_mem_copy_cmd_len(gt_to_xe(gt)); + + bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type | (len - 2); bb->cs[bb->len++] = width - 1; bb->cs[bb->len++] = size / pitch - 1; /* ignored by hw for page-copy/linear above */ bb->cs[bb->len++] = pitch - 1; @@ -966,7 +987,7 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m, } /* Add copy commands size here */ - batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + + batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) + ((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0)); bb = xe_bb_new(gt, batch_size, usm); @@ -1145,6 +1166,8 @@ static int emit_flush_invalidate(u32 *dw, int i, u32 flags) * @tile: Tile whose migration context to be used. * @q : Execution to be used along with migration context. * @src_bo: The buffer object @src is currently bound to. + * @new_mem: The (not yet committed) destination resource @src_bo is being + * moved into; src_bo->ttm.resource is still the old resource. * @read_write : Creates BB commands for CCS read/write. * * Creates batch buffer instructions to copy CCS metadata from CCS pool to @@ -1156,12 +1179,13 @@ static int emit_flush_invalidate(u32 *dw, int i, u32 flags) */ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, struct xe_bo *src_bo, + struct ttm_resource *new_mem, enum xe_sriov_vf_ccs_rw_ctxs read_write) { bool src_is_pltt = read_write == XE_SRIOV_VF_CCS_READ_CTX; bool dst_is_pltt = read_write == XE_SRIOV_VF_CCS_WRITE_CTX; - struct ttm_resource *src = src_bo->ttm.resource; + struct ttm_resource *src = new_mem; struct xe_migrate *m = tile->migrate; struct xe_gt *gt = tile->primary_gt; u32 batch_size, batch_size_allocated; @@ -1405,7 +1429,7 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off batch_size += pte_update_size(m, 0, sysmem, &sysmem_it, &vram_L0, &sysmem_L0_ofs, &sysmem_L0_pt, 0, avail_pts, avail_pts); - batch_size += EMIT_COPY_DW; + batch_size += emit_copy_cmd_len(xe); bb = xe_bb_new(gt, batch_size, usm); if (IS_ERR(bb)) { @@ -1460,12 +1484,17 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off return fence; } +static u32 blt_mem_set_cmd_len(struct xe_device *xe) +{ + return 7; +} + static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch) { struct xe_device *xe = gt_to_xe(gt); u32 *cs = bb->cs + bb->len; - u32 len = PVC_MEM_SET_CMD_LEN_DW; + u32 len = blt_mem_set_cmd_len(xe); *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); *cs++ = pitch - 1; @@ -1483,15 +1512,21 @@ static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs bb->len += len; } +static u32 blt_fast_color_cmd_len(struct xe_device *xe) +{ + if (GRAPHICS_VERx100(xe) >= 1250) + return 16; + else + return 11; +} + static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch, bool is_vram) { struct xe_device *xe = gt_to_xe(gt); u32 *cs = bb->cs + bb->len; - u32 len = XY_FAST_COLOR_BLT_DW; + u32 len = blt_fast_color_cmd_len(xe); - if (GRAPHICS_VERx100(xe) < 1250) - len = 11; *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | (len - 2); @@ -1526,10 +1561,12 @@ static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, static u32 emit_clear_cmd_len(struct xe_gt *gt) { + struct xe_device *xe = gt_to_xe(gt); + if (gt->info.has_xe2_blt_instructions) - return PVC_MEM_SET_CMD_LEN_DW; + return blt_mem_set_cmd_len(xe); else - return XY_FAST_COLOR_BLT_DW; + return blt_fast_color_cmd_len(xe); } static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, @@ -1862,7 +1899,7 @@ __xe_migrate_update_pgtables(struct xe_migrate *m, /* For sysmem PTE's, need to map them in our hole.. */ if (!IS_DGFX(xe)) { - u16 pat_index = xe->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); u32 ptes, ofs; ppgtt_ofs = NUM_KERNEL_PDE - 1; @@ -2084,7 +2121,7 @@ static void build_pt_update_batch_sram(struct xe_migrate *m, struct drm_pagemap_addr *sram_addr, u32 size, int level) { - u16 pat_index = tile_to_xe(m->tile)->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(m->tile), XE_CACHE_WB); u64 gpu_page_size = 0x1ull << xe_pt_shift(level); u32 ptes; int i = 0; @@ -2202,7 +2239,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, xe_assert(xe, npages * PAGE_SIZE <= MAX_PREEMPTDISABLE_TRANSFER); batch_size += pte_update_cmd_size(npages << PAGE_SHIFT); - batch_size += EMIT_COPY_DW; + batch_size += emit_copy_cmd_len(xe); bb = xe_bb_new(gt, batch_size, use_usm_batch); if (IS_ERR(bb)) { diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 169279d9d8c2..78e5b63f3ebe 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -3,8 +3,8 @@ * Copyright © 2020 Intel Corporation */ -#ifndef _XE_MIGRATE_ -#define _XE_MIGRATE_ +#ifndef _XE_MIGRATE_H_ +#define _XE_MIGRATE_H_ #include <linux/types.h> @@ -138,6 +138,7 @@ struct dma_fence *xe_migrate_resolve(struct xe_migrate *m, int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, struct xe_bo *src_bo, + struct ttm_resource *new_mem, enum xe_sriov_vf_ccs_rw_ctxs read_write); void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c index 4cb578182912..99347f216ec8 100644 --- a/drivers/gpu/drm/xe/xe_module.c +++ b/drivers/gpu/drm/xe/xe_module.c @@ -7,6 +7,7 @@ #include <linux/init.h> #include <linux/module.h> +#include <linux/workqueue.h> #include <drm/drm_module.h> @@ -91,6 +92,50 @@ static int xe_check_nomodeset(void) return 0; } +static struct workqueue_struct *xe_destroy_wq; + +static int __init xe_destroy_wq_module_init(void) +{ + xe_destroy_wq = alloc_workqueue("xe-guc-destroy-wq", WQ_UNBOUND, 0); + if (!xe_destroy_wq) + return -ENOMEM; + return 0; +} + +static void xe_destroy_wq_module_exit(void) +{ + if (xe_destroy_wq) + destroy_workqueue(xe_destroy_wq); + xe_destroy_wq = NULL; +} + +/** + * xe_destroy_wq_queue() - Queue work on the destroy workqueue + * @work: work item to queue + * + * The destroy workqueue has module lifetime and is used for GuC exec queue + * teardown that can outlive a single xe_device. SVM pagemap destroy uses the + * per-device xe->destroy_wq instead. + * + * Return: %true if @work was queued, %false if it was already pending. + */ +bool xe_destroy_wq_queue(struct work_struct *work) +{ + return queue_work(xe_destroy_wq, work); +} + +/** + * xe_destroy_wq_flush() - Flush the destroy workqueue + * + * Drains all pending destroy work. Called from PCI remove to ensure + * teardown ordering before the device is destroyed. + */ +void xe_destroy_wq_flush(void) +{ + if (xe_destroy_wq) + flush_workqueue(xe_destroy_wq); +} + struct init_funcs { int (*init)(void); void (*exit)(void); @@ -113,6 +158,10 @@ static const struct init_funcs init_funcs[] = { .exit = xe_sched_job_module_exit, }, { + .init = xe_destroy_wq_module_init, + .exit = xe_destroy_wq_module_exit, + }, + { .init = xe_register_pci_driver, .exit = xe_unregister_pci_driver, }, diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h index 79cb9639c0f3..e8e54f701cf3 100644 --- a/drivers/gpu/drm/xe/xe_module.h +++ b/drivers/gpu/drm/xe/xe_module.h @@ -8,6 +8,8 @@ #include <linux/types.h> +struct work_struct; + /* Module modprobe variables */ struct xe_modparam { bool force_execlist; @@ -27,5 +29,8 @@ struct xe_modparam { extern struct xe_modparam xe_modparam; +bool xe_destroy_wq_queue(struct work_struct *work); +void xe_destroy_wq_flush(void); + #endif diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c index 33487e91f366..1ea67eaeae24 100644 --- a/drivers/gpu/drm/xe/xe_nvm.c +++ b/drivers/gpu/drm/xe/xe_nvm.c @@ -60,35 +60,40 @@ static bool xe_nvm_writable_override(struct xe_device *xe) struct xe_mmio *mmio = xe_root_tile_mmio(xe); bool writable_override; struct xe_reg reg; - u32 test_bit; + u32 test_bit, test_val; switch (xe->info.platform) { case XE_CRESCENTISLAND: reg = PCODE_SCRATCH(0); test_bit = FDO_MODE; + test_val = FDO_MODE; break; case XE_BATTLEMAGE: reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE); test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; + test_val = 0; break; case XE_PVC: reg = HECI_FWSTS2(PVC_GSC_HECI2_BASE); test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; + test_val = 0; break; case XE_DG2: reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE); test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; + test_val = 0; break; case XE_DG1: reg = HECI_FWSTS2(DG1_GSC_HECI2_BASE); test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE; + test_val = 0; break; default: drm_err(&xe->drm, "Unknown platform\n"); return true; } - writable_override = !(xe_mmio_read32(mmio, reg) & test_bit); + writable_override = (xe_mmio_read32(mmio, reg) & test_bit) == test_val; if (writable_override) drm_info(&xe->drm, "NVM access overridden by jumper\n"); return writable_override; diff --git a/drivers/gpu/drm/xe/xe_nvm.h b/drivers/gpu/drm/xe/xe_nvm.h index fd3467ad35a4..b14722103f81 100644 --- a/drivers/gpu/drm/xe/xe_nvm.h +++ b/drivers/gpu/drm/xe/xe_nvm.h @@ -3,8 +3,8 @@ * Copyright(c) 2019-2025 Intel Corporation. All rights reserved. */ -#ifndef __XE_NVM_H__ -#define __XE_NVM_H__ +#ifndef _XE_NVM_H_ +#define _XE_NVM_H_ struct xe_device; diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index d908f4e03906..2dce6a47202c 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -9,11 +9,13 @@ #include <linux/poll.h> #include <drm/drm_drv.h> +#include <drm/drm_gem.h> #include <drm/drm_managed.h> #include <drm/drm_syncobj.h> #include <uapi/drm/xe_drm.h> #include <generated/xe_wa_oob.h> +#include <generated/xe_device_wa_oob.h> #include "abi/guc_actions_slpc_abi.h" #include "instructions/xe_mi_commands.h" @@ -35,6 +37,7 @@ #include "xe_oa.h" #include "xe_observation.h" #include "xe_pm.h" +#include "xe_reg_whitelist.h" #include "xe_sched_job.h" #include "xe_sriov.h" #include "xe_sync.h" @@ -213,32 +216,45 @@ static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream) #define oa_report_header_64bit(__s) \ ((__s)->oa_buffer.format->header == HDR_64_BIT) -static u64 oa_report_id(struct xe_oa_stream *stream, void *report) +static u64 oa_report_id(struct xe_oa_stream *stream, u32 report_offset) { - return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report; + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + return oa_report_header_64bit(stream) ? + xe_map_rd(stream->oa->xe, map, report_offset, u64) : + xe_map_rd(stream->oa->xe, map, report_offset, u32); } -static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report) +static void oa_report_id_clear(struct xe_oa_stream *stream, u32 report_offset) { - if (oa_report_header_64bit(stream)) - *(u64 *)report = 0; - else - *report = 0; + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + oa_report_header_64bit(stream) ? + xe_map_wr(stream->oa->xe, map, report_offset, u64, 0) : + xe_map_wr(stream->oa->xe, map, report_offset, u32, 0); } -static u64 oa_timestamp(struct xe_oa_stream *stream, void *report) +static u64 oa_timestamp(struct xe_oa_stream *stream, u32 report_offset) { + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + return oa_report_header_64bit(stream) ? - *((u64 *)report + 1) : - *((u32 *)report + 1); + xe_map_rd(stream->oa->xe, map, report_offset + 8, u64) : + xe_map_rd(stream->oa->xe, map, report_offset + 4, u32); } -static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report) +static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 report_offset) { - if (oa_report_header_64bit(stream)) - *(u64 *)&report[2] = 0; - else - report[1] = 0; + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + oa_report_header_64bit(stream) ? + xe_map_wr(stream->oa->xe, map, report_offset + 8, u64, 0) : + xe_map_wr(stream->oa->xe, map, report_offset + 4, u32, 0); +} + +static bool mert_wa_14026633728(struct xe_oa_stream *s) +{ + return s->oa_unit->type == DRM_XE_OA_UNIT_TYPE_MERT && XE_DEVICE_WA(s->oa->xe, 14026633728); } static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) @@ -275,9 +291,7 @@ static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) * they were written. If not : (╯°□°）╯︵ ┻━┻ */ while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) { - void *report = stream->oa_buffer.vaddr + tail; - - if (oa_report_id(stream, report) || oa_timestamp(stream, report)) + if (oa_report_id(stream, tail) || oa_timestamp(stream, tail)) break; tail = xe_oa_circ_diff(stream, tail, report_size); @@ -311,30 +325,37 @@ static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer) return HRTIMER_RESTART; } +static unsigned long +xe_oa_copy_to_user(struct xe_oa_stream *stream, void __user *dst, u32 report_offset, u32 len) +{ + xe_assert(stream->oa->xe, len <= stream->oa_buffer.format->size); + + xe_map_memcpy_from(stream->oa->xe, stream->oa_buffer.bounce, + &stream->oa_buffer.bo->vmap, report_offset, len); + return copy_to_user(dst, stream->oa_buffer.bounce, len); +} + static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf, - size_t count, size_t *offset, const u8 *report) + size_t count, size_t *offset, u32 report_offset) { int report_size = stream->oa_buffer.format->size; int report_size_partial; - u8 *oa_buf_end; if ((count - *offset) < report_size) return -ENOSPC; buf += *offset; - oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; - report_size_partial = oa_buf_end - report; + report_size_partial = stream->oa_buffer.circ_size - report_offset; if (report_size_partial < report_size) { - if (copy_to_user(buf, report, report_size_partial)) + if (xe_oa_copy_to_user(stream, buf, report_offset, report_size_partial)) return -EFAULT; buf += report_size_partial; - if (copy_to_user(buf, stream->oa_buffer.vaddr, - report_size - report_size_partial)) + if (xe_oa_copy_to_user(stream, buf, 0, report_size - report_size_partial)) return -EFAULT; - } else if (copy_to_user(buf, report, report_size)) { + } else if (xe_oa_copy_to_user(stream, buf, report_offset, report_size)) { return -EFAULT; } @@ -347,7 +368,6 @@ static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, size_t count, size_t *offset) { int report_size = stream->oa_buffer.format->size; - u8 *oa_buf_base = stream->oa_buffer.vaddr; u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); size_t start_offset = *offset; unsigned long flags; @@ -364,26 +384,24 @@ static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, for (; xe_oa_circ_diff(stream, tail, head); head = xe_oa_circ_incr(stream, head, report_size)) { - u8 *report = oa_buf_base + head; - - ret = xe_oa_append_report(stream, buf, count, offset, report); + ret = xe_oa_append_report(stream, buf, count, offset, head); if (ret) break; if (!(stream->oa_buffer.circ_size % report_size)) { /* Clear out report id and timestamp to detect unlanded reports */ - oa_report_id_clear(stream, (void *)report); - oa_timestamp_clear(stream, (void *)report); + oa_report_id_clear(stream, head); + oa_timestamp_clear(stream, head); } else { - u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; - u32 part = oa_buf_end - report; + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + u32 part = stream->oa_buffer.circ_size - head; /* Zero out the entire report */ if (report_size <= part) { - memset(report, 0, report_size); + xe_map_memset(stream->oa->xe, map, head, 0, report_size); } else { - memset(report, 0, part); - memset(oa_buf_base, 0, report_size - part); + xe_map_memset(stream->oa->xe, map, head, 0, part); + xe_map_memset(stream->oa->xe, map, 0, 0, report_size - part); } } } @@ -436,7 +454,8 @@ static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream) spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); /* Zero out the OA buffer since we rely on zero report id and timestamp fields */ - memset(stream->oa_buffer.vaddr, 0, xe_bo_size(stream->oa_buffer.bo)); + xe_map_memset(stream->oa->xe, &stream->oa_buffer.bo->vmap, 0, 0, + xe_bo_size(stream->oa_buffer.bo)); } static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask) @@ -699,6 +718,7 @@ static int num_lri_dwords(int num_regs) static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream) { xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); + kfree(stream->oa_buffer.bounce); } static void xe_oa_free_configs(struct xe_oa_stream *stream) @@ -866,6 +886,9 @@ static void xe_oa_stream_destroy(struct xe_oa_stream *stream) mutex_destroy(&stream->stream_lock); + if (stream->sample) + xe_reg_dewhitelist_oa_regs(stream->gt); + xe_oa_disable_metric_set(stream); xe_exec_queue_put(stream->k_exec_q); @@ -880,18 +903,25 @@ static void xe_oa_stream_destroy(struct xe_oa_stream *stream) static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size) { + u32 vram = mert_wa_14026633728(stream) ? + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(stream->oa->xe)) : + XE_BO_FLAG_SYSTEM; struct xe_bo *bo; bo = xe_bo_create_pin_map_novm(stream->oa->xe, stream->gt->tile, size, ttm_bo_type_kernel, - XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT, false); + vram | XE_BO_FLAG_GGTT, false); if (IS_ERR(bo)) return PTR_ERR(bo); stream->oa_buffer.bo = bo; - /* mmap implementation requires OA buffer to be in system memory */ - xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0); - stream->oa_buffer.vaddr = bo->vmap.vaddr; + + stream->oa_buffer.bounce = kmalloc(stream->oa_buffer.format->size, GFP_KERNEL); + if (!stream->oa_buffer.bounce) { + xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); + return -ENOMEM; + } + return 0; } @@ -1673,8 +1703,6 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) { struct xe_oa_stream *stream = file->private_data; struct xe_bo *bo = stream->oa_buffer.bo; - unsigned long start = vma->vm_start; - int i, ret; if (xe_observation_paranoid && !perfmon_capable()) { drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); @@ -1682,7 +1710,7 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) } /* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */ - if (vma->vm_end - vma->vm_start != xe_bo_size(stream->oa_buffer.bo)) { + if (vma->vm_end - vma->vm_start != xe_bo_size(bo)) { drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n"); return -EINVAL; } @@ -1698,17 +1726,7 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY, VM_MAYWRITE | VM_MAYEXEC); - xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma)); - for (i = 0; i < bo->ttm.ttm->num_pages; i++) { - ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]), - PAGE_SIZE, vma->vm_page_prot); - if (ret) - break; - - start += PAGE_SIZE; - } - - return ret; + return drm_gem_mmap_obj(&bo->ttm.base, xe_bo_size(bo), vma); } static const struct file_operations xe_oa_fops = { @@ -1871,6 +1889,9 @@ static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa, goto err_disable; } + if (stream->sample) + xe_reg_whitelist_oa_regs(stream->gt); + /* Hold a reference on the drm device till stream_fd is released */ drm_dev_get(&stream->oa->xe->drm); @@ -1920,16 +1941,21 @@ static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent) return div_u64(nom + den - 1, den); } -static bool oa_unit_supports_oa_format(struct xe_oa_open_param *param, int type) +static bool oa_unit_supports_oa_format(struct xe_oa *oa, struct xe_oa_open_param *param) { + const struct xe_oa_format *f = &oa->oa_formats[param->oa_format]; + switch (param->oa_unit->type) { case DRM_XE_OA_UNIT_TYPE_OAG: - return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR || - type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC; + return f->type == DRM_XE_OA_FMT_TYPE_OAG || f->type == DRM_XE_OA_FMT_TYPE_OAR || + f->type == DRM_XE_OA_FMT_TYPE_OAC || f->type == DRM_XE_OA_FMT_TYPE_PEC; + case DRM_XE_OA_UNIT_TYPE_MERT: + if (XE_DEVICE_WA(oa->xe, 14026746987)) + return param->oa_format == XE_OAM_FORMAT_MPEC8u32_B8_C8; + fallthrough; case DRM_XE_OA_UNIT_TYPE_OAM: case DRM_XE_OA_UNIT_TYPE_OAM_SAG: - case DRM_XE_OA_UNIT_TYPE_MERT: - return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; + return f->type == DRM_XE_OA_FMT_TYPE_OAM || f->type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; default: return false; } @@ -2071,8 +2097,7 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f goto err_exec_q; f = &oa->oa_formats[param.oa_format]; - if (!param.oa_format || !f->size || - !oa_unit_supports_oa_format(¶m, f->type)) { + if (!param.oa_format || !f->size || !oa_unit_supports_oa_format(oa, ¶m)) { drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n", param.oa_format, f->type, f->size, param.hwe->class); ret = -EINVAL; @@ -2233,15 +2258,19 @@ static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr) return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs); } -static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr) +static bool xe_oa_is_valid_config_reg(struct xe_oa *oa, u32 addr, u32 val) { + if (XE_DEVICE_WA(oa->xe, 14026779378) && + addr == SYS_MEM_LAT_MEASURE.addr && val & SYS_MEM_LAT_MEASURE_EN) + return false; + return xe_oa_is_valid_flex_addr(oa, addr) || xe_oa_is_valid_b_counter_addr(oa, addr) || xe_oa_is_valid_mux_addr(oa, addr); } static struct xe_oa_reg * -xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr), +xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr, u32 val), u32 __user *regs, u32 n_regs) { struct xe_oa_reg *oa_regs; @@ -2259,16 +2288,16 @@ xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr), if (err) goto addr_err; - if (!is_valid(oa, addr)) { - drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr); - err = -EINVAL; - goto addr_err; - } - err = get_user(value, regs + 1); if (err) goto addr_err; + if (!is_valid(oa, addr, value)) { + drm_dbg(&oa->xe->drm, "Invalid oa_reg addr/value: %#x %#x\n", addr, value); + err = -EINVAL; + goto addr_err; + } + oa_regs[i].addr = XE_REG(addr); oa_regs[i].value = value; @@ -2367,7 +2396,7 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid)); oa_config->regs_len = arg->n_regs; - regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr, + regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg, u64_to_user_ptr(arg->regs_ptr), arg->n_regs); if (IS_ERR(regs)) { diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h index b03ffd513483..e876e9be92ba 100644 --- a/drivers/gpu/drm/xe/xe_oa_types.h +++ b/drivers/gpu/drm/xe/xe_oa_types.h @@ -67,7 +67,7 @@ struct xe_oa_format { u32 counter_select; /** @size: record size as written by HW (multiple of 64 byte cachelines) */ int size; - /** @type: of enum @drm_xe_oa_format_type */ + /** @type: of enum drm_xe_oa_format_type */ int type; /** @header: 32 or 64 bit report headers */ enum xe_oa_report_header header; @@ -126,6 +126,9 @@ struct xe_oa_gt { /** @oa_unit: array of oa_units */ struct xe_oa_unit *oa_unit; + + /** @whitelist_count: number of open streams for which oa registers are whitelisted */ + u32 whitelist_count; }; /** @@ -154,16 +157,18 @@ struct xe_oa { u16 oa_unit_ids; }; -/** @xe_oa_buffer: State of the stream OA buffer */ +/** + * struct xe_oa_buffer - State of the stream OA buffer + */ struct xe_oa_buffer { /** @format: data format */ const struct xe_oa_format *format; - /** @format: xe_bo backing the OA buffer */ + /** @bo: xe_bo backing the OA buffer */ struct xe_bo *bo; - /** @vaddr: mapped vaddr of the OA buffer */ - u8 *vaddr; + /** @bounce: bounce buffer used with xe_map layer */ + void *bounce; /** @ptr_lock: Lock protecting reads/writes to head/tail pointers */ spinlock_t ptr_lock; diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h index b3289219b1be..c4ee625b93dd 100644 --- a/drivers/gpu/drm/xe/xe_pagefault_types.h +++ b/drivers/gpu/drm/xe/xe_pagefault_types.h @@ -86,7 +86,7 @@ struct xe_pagefault { u8 engine_class; /** @consumer.engine_instance: engine instance */ u8 engine_instance; - /** consumer.reserved: reserved bits for future expansion */ + /** @consumer.reserved: reserved bits for future expansion */ u64 reserved; } consumer; /** @@ -112,7 +112,7 @@ struct xe_pagefault { }; /** - * struct xe_pagefault_queue: Xe pagefault queue (consumer) + * struct xe_pagefault_queue - Xe pagefault queue (consumer) * * Used to capture all device page faults for deferred processing. Size this * queue to absorb the device’s worst-case number of outstanding faults. diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 356f53bdb83c..fad5b5a5ed4a 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -531,6 +531,14 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) drm_printf(p, "Page Table Access:\n"); xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); + if (xe_gt_is_media_type(gt)) + pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_ATS)); + else + pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_ATS)); + + drm_printf(p, "PCIe ATS/PASID:\n"); + xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); + return 0; } @@ -551,6 +559,7 @@ static const struct xe_pat_ops xe3p_xpc_pat_ops = { void xe_pat_init_early(struct xe_device *xe) { xe->pat.idx[XE_CACHE_WB_COMPRESSION] = XE_PAT_INVALID_IDX; + xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = XE_PAT_INVALID_IDX; if (GRAPHICS_VERx100(xe) == 3511) { xe->pat.ops = &xe3p_xpc_pat_ops; xe->pat.table = xe3p_xpc_pat_table; diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h index a1e287c08f57..7060f66e1d63 100644 --- a/drivers/gpu/drm/xe/xe_pat.h +++ b/drivers/gpu/drm/xe/xe_pat.h @@ -82,4 +82,12 @@ bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index); */ u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index); +#define xe_cache_pat_idx(xe, cache_mode) ({ \ + const struct xe_device *__xedev = (xe); \ + enum xe_cache_level __mode = (cache_mode); \ + xe_assert(__xedev, __mode < __XE_CACHE_LEVEL_COUNT); \ + xe_assert(__xedev, __xedev->pat.idx[__mode] != XE_PAT_INVALID_IDX); \ + __xedev->pat.idx[__mode]; \ +}) + #endif diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index c2ecd27ec770..9cd873708136 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -444,6 +444,7 @@ static const struct xe_device_desc nvls_desc = { .has_display = true, .has_flat_ccs = 1, .has_pre_prod_wa = 1, + .has_sriov = true, .max_gt_per_tile = 2, MULTI_LRC_MASK, .require_force_probe = true, @@ -466,6 +467,7 @@ static const struct xe_device_desc cri_desc = { .has_soc_remapper_sysctrl = true, .has_soc_remapper_telem = true, .has_sriov = true, + .has_sysctrl = true, .max_gt_per_tile = 2, MULTI_LRC_MASK, .require_force_probe = true, @@ -481,6 +483,7 @@ static const struct xe_device_desc nvlp_desc = { .has_flat_ccs = 1, .has_page_reclaim_hw_assist = true, .has_pre_prod_wa = true, + .has_sriov = true, .max_gt_per_tile = 2, MULTI_LRC_MASK, .require_force_probe = true, @@ -572,14 +575,14 @@ static bool id_blocked(u16 device_id) } static const struct xe_subplatform_desc * -find_subplatform(const struct xe_device *xe, const struct xe_device_desc *desc) +find_subplatform(const struct xe_device_desc *desc, u16 devid) { const struct xe_subplatform_desc *sp; const u16 *id; for (sp = desc->subplatforms; sp && sp->subplatform; sp++) for (id = sp->pciidlist; *id; id++) - if (*id == xe->info.devid) + if (*id == devid) return sp; return NULL; @@ -723,6 +726,16 @@ static int handle_gmdid(struct xe_device *xe, return 0; } +static void init_devid(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + KUNIT_STATIC_STUB_REDIRECT(init_devid, xe); + + xe->info.devid = pdev->device; + xe->info.revid = pdev->revision; +} + /* * Initialize device info content that only depends on static driver_data * passed to the driver at probe time from PCI ID table. @@ -738,6 +751,8 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.subplatform = subplatform_desc ? subplatform_desc->subplatform : XE_SUBPLATFORM_NONE; + init_devid(xe); + xe->info.dma_mask_size = desc->dma_mask_size; xe->info.va_bits = desc->va_bits; xe->info.vm_max_level = desc->vm_max_level; @@ -764,8 +779,8 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem; xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && desc->has_sriov; + xe->info.has_sysctrl = desc->has_sysctrl; xe->info.skip_guc_pc = desc->skip_guc_pc; - xe->info.skip_mtcfg = desc->skip_mtcfg; xe->info.skip_pcode = desc->skip_pcode; xe->info.needs_scratch = desc->needs_scratch; xe->info.needs_shared_vf_gt_wq = desc->needs_shared_vf_gt_wq; @@ -773,7 +788,9 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.probe_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) && xe_modparam.probe_display && - desc->has_display; + desc->has_display && + !xe_device_is_admin_only(xe); + xe->info.force_execlist = xe_modparam.force_execlist; xe_assert(xe, desc->max_gt_per_tile > 0); xe_assert(xe, desc->max_gt_per_tile <= XE_MAX_GT_PER_TILE); @@ -807,9 +824,6 @@ static void xe_info_probe_tile_count(struct xe_device *xe) if (xe->info.tile_count == 1) return; - if (xe->info.skip_mtcfg) - return; - mmio = xe_root_tile_mmio(xe); /* @@ -969,6 +983,12 @@ static int xe_info_init(struct xe_device *xe, xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp; xe->info.has_mem_copy_instr = GRAPHICS_VER(xe) >= 20; + if (IS_SRIOV_VF(xe)) { + xe->info.has_sysctrl = 0; + xe->info.has_soc_remapper_sysctrl = 0; + xe->info.has_soc_remapper_telem = 0; + } + xe_info_probe_tile_count(xe); for_each_remote_tile(tile, xe, id) { @@ -1029,6 +1049,12 @@ static void xe_pci_remove(struct pci_dev *pdev) return; xe_device_remove(xe); + + /* + * Preserve remove-time flush after moving destroy work to module + * lifetime. + */ + xe_destroy_wq_flush(); xe_pm_fini(xe); } @@ -1058,6 +1084,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct xe_device *xe; int err; + subplatform_desc = find_subplatform(desc, pdev->device); + xe_configfs_check_device(pdev); if (desc->require_force_probe && !id_forced(pdev->device)) { @@ -1085,14 +1113,13 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) return err; - xe = xe_device_create(pdev, ent); + xe = xe_device_create(pdev); if (IS_ERR(xe)) return PTR_ERR(xe); pci_set_drvdata(pdev, &xe->drm); xe_pm_assert_unbounded_bridge(xe); - subplatform_desc = find_subplatform(xe, desc); pci_set_master(pdev); @@ -1147,7 +1174,7 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) str_yes_no(xe_device_has_sriov(xe)), xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); - err = xe_pm_init_early(xe); + err = xe_pm_probe(xe); if (err) return err; @@ -1159,9 +1186,6 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) goto err_driver_cleanup; - drm_dbg(&xe->drm, "d3cold: capable=%s\n", - str_yes_no(xe->d3cold.capable)); - return 0; err_driver_cleanup: diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index 3fd22034f03e..35e6b53e18ce 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -191,6 +191,8 @@ static int pf_disable_vfs(struct xe_device *xe) pci_disable_sriov(pdev); + xe_sriov_pf_reprovision_default(xe); + pf_reset_vfs(xe, num_vfs); xe_sriov_pf_unprovision_vfs(xe, num_vfs); diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index 8eee4fb1c57c..5b85e2c24b7b 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -57,9 +57,9 @@ struct xe_device_desc { u8 has_soc_remapper_sysctrl:1; u8 has_soc_remapper_telem:1; u8 has_sriov:1; + u8 has_sysctrl:1; u8 needs_scratch:1; u8 skip_guc_pc:1; - u8 skip_mtcfg:1; u8 skip_pcode:1; u8 needs_shared_vf_gt_wq:1; }; diff --git a/drivers/gpu/drm/xe/xe_pcode_api.h b/drivers/gpu/drm/xe/xe_pcode_api.h index 85cc7478b787..94575c476e3d 100644 --- a/drivers/gpu/drm/xe/xe_pcode_api.h +++ b/drivers/gpu/drm/xe/xe_pcode_api.h @@ -3,6 +3,9 @@ * Copyright © 2022 Intel Corporation */ +#ifndef _XE_PCODE_API_H_ +#define _XE_PCODE_API_H_ + /* Internal to xe_pcode */ #include "regs/xe_reg_defs.h" @@ -47,8 +50,9 @@ #define WRITE_PSYSGPU_POWER_LIMIT 0x7 #define READ_PACKAGE_POWER_LIMIT 0x8 #define WRITE_PACKAGE_POWER_LIMIT 0x9 -#define READ_PL_FROM_FW 0x1 #define READ_PL_FROM_PCODE 0x0 +#define READ_PL_FROM_FW 0x1 +#define READ_PL_ACCEPTED 0x2 #define PCODE_THERMAL_INFO 0x25 #define READ_THERMAL_LIMITS 0x0 @@ -101,3 +105,5 @@ #define BMG_PCIE_CAP XE_REG(0x138340) #define LINK_DOWNGRADE REG_GENMASK(1, 0) #define DOWNGRADE_CAPABLE 2 + +#endif diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index 01185f10a883..99562f691080 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -24,8 +24,10 @@ #include "xe_irq.h" #include "xe_late_bind_fw.h" #include "xe_pcode.h" +#include "xe_printk.h" #include "xe_pxp.h" #include "xe_sriov_vf_ccs.h" +#include "xe_sysctrl.h" #include "xe_trace.h" #include "xe_vm.h" #include "xe_wa.h" @@ -259,6 +261,8 @@ int xe_pm_resume(struct xe_device *xe) xe_i2c_pm_resume(xe, true); + xe_sysctrl_pm_resume(xe); + xe_irq_resume(xe); for_each_gt(gt, xe, id) { @@ -346,10 +350,22 @@ static void xe_pm_runtime_init(struct xe_device *xe) pm_runtime_put(dev); } +/** + * xe_pm_init_early() - Initialize Xe Power Management + * @xe: the &xe_device instance + * + * Initialize everything that is a "software-only" state that does not + * require access to any of the device's hardware data. + * + * Return: 0 on success or a negative error code on failure. + */ int xe_pm_init_early(struct xe_device *xe) { int err; + init_completion(&xe->pm_block); + complete_all(&xe->pm_block); + INIT_LIST_HEAD(&xe->rebind_resume_list); INIT_LIST_HEAD(&xe->mem_access.vram_userfault.list); err = drmm_mutex_init(&xe->drm, &xe->mem_access.vram_userfault.lock); @@ -360,11 +376,30 @@ int xe_pm_init_early(struct xe_device *xe) if (err) return err; - xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe); + err = drmm_mutex_init(&xe->drm, &xe->rebind_resume_lock); + if (err) + return err; + return 0; } ALLOW_ERROR_INJECTION(xe_pm_init_early, ERRNO); /* See xe_pci_probe() */ +/** + * xe_pm_probe() - Initialize Xe Power Management + * @xe: the &xe_device instance + * + * Check d3cold capability. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_pm_probe(struct xe_device *xe) +{ + xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe); + xe_dbg(xe, "d3cold: capable=%s\n", str_yes_no(xe->d3cold.capable)); + + return 0; +} + static u32 vram_threshold_value(struct xe_device *xe) { if (xe->info.platform == XE_BATTLEMAGE) { @@ -456,14 +491,6 @@ int xe_pm_init(struct xe_device *xe) if (err) return err; - err = drmm_mutex_init(&xe->drm, &xe->rebind_resume_lock); - if (err) - goto err_unregister; - - init_completion(&xe->pm_block); - complete_all(&xe->pm_block); - INIT_LIST_HEAD(&xe->rebind_resume_list); - /* For now suspend/resume is only allowed with GuC */ if (!xe_device_uc_enabled(xe)) return 0; @@ -670,6 +697,9 @@ int xe_pm_runtime_resume(struct xe_device *xe) xe_i2c_pm_resume(xe, xe->d3cold.allowed); + if (xe->d3cold.allowed) + xe_sysctrl_pm_resume(xe); + xe_irq_resume(xe); for_each_gt(gt, xe, id) { diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index 6b27039e7b2d..6d5ab09cb769 100644 --- a/drivers/gpu/drm/xe/xe_pm.h +++ b/drivers/gpu/drm/xe/xe_pm.h @@ -17,6 +17,7 @@ int xe_pm_suspend(struct xe_device *xe); int xe_pm_resume(struct xe_device *xe); int xe_pm_init_early(struct xe_device *xe); +int xe_pm_probe(struct xe_device *xe); int xe_pm_init(struct xe_device *xe); void xe_pm_fini(struct xe_device *xe); bool xe_pm_runtime_suspended(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 8e5f4f0dea3f..39c9c8f0ea2d 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -14,6 +14,7 @@ #include "xe_gt_stats.h" #include "xe_migrate.h" #include "xe_page_reclaim.h" +#include "xe_pat.h" #include "xe_pt_types.h" #include "xe_pt_walk.h" #include "xe_res_cursor.h" @@ -62,7 +63,7 @@ static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm, unsigned int level) { struct xe_device *xe = tile_to_xe(tile); - u16 pat_index = xe->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); u8 id = tile->id; if (!xe_vm_has_scratch(vm)) @@ -432,6 +433,7 @@ xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, struct xe_pt_stage_bind_walk *xe_walk) { + struct xe_bo *bo = xe_vma_bo(xe_walk->vma); u64 size, dma; if (level > MAX_HUGEPTE_LEVEL) @@ -445,8 +447,8 @@ static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, if (next - xe_walk->va_curs_start > xe_walk->curs->size) return false; - /* null VMA's do not have dma addresses */ - if (xe_vma_is_null(xe_walk->vma)) + /* null VMA's and purged BO's do not have dma addresses */ + if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo))) return true; /* if we are clearing page table, no dma addresses*/ @@ -467,6 +469,7 @@ static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, static bool xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) { + struct xe_bo *bo = xe_vma_bo(xe_walk->vma); struct xe_res_cursor curs = *xe_walk->curs; if (!IS_ALIGNED(addr, SZ_64K)) @@ -475,8 +478,8 @@ xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) if (next > xe_walk->l0_end_addr) return false; - /* null VMA's do not have dma addresses */ - if (xe_vma_is_null(xe_walk->vma)) + /* null VMA's and purged BO's do not have dma addresses */ + if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo))) return true; xe_res_next(&curs, addr - xe_walk->va_curs_start); @@ -707,7 +710,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, { struct xe_device *xe = tile_to_xe(tile); struct xe_bo *bo = xe_vma_bo(vma); - struct xe_res_cursor curs; + struct xe_res_cursor curs = {}; struct xe_vm *vm = xe_vma_vm(vma); struct xe_pt_stage_bind_walk xe_walk = { .base = { @@ -884,13 +887,21 @@ static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, { struct xe_pt_zap_ptes_walk *xe_walk = container_of(walk, typeof(*xe_walk), base); - struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); + struct xe_pt *xe_child; pgoff_t end_offset; - XE_WARN_ON(!*child); XE_WARN_ON(!level); /* + * Below would be unexpected behavior that needs to be root caused + * but better warn and bail than crash the driver. + */ + if (XE_WARN_ON(!*child)) + return 0; + + xe_child = container_of(*child, typeof(*xe_child), base); + + /* * Note that we're called from an entry callback, and we're dealing * with the child of that entry rather than the parent, so need to * adjust level down. @@ -1015,12 +1026,22 @@ xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *t u64 *ptr = data; u32 i; + /* + * @qword_ofs is the absolute entry offset within the page table, while + * @ptes is indexed relative to @update->ofs (its first entry). The GPU + * path (write_pgtable) splits a single update into MAX_PTE_PER_SDI-sized + * chunks, calling this with an advancing @qword_ofs but a fresh @data + * pointer per chunk, so translate back into a @ptes index rather than + * assuming the chunk starts at ptes[0]. + */ for (i = 0; i < num_qwords; i++) { + u32 idx = qword_ofs - update->ofs + i; + if (map) xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * - sizeof(u64), u64, ptes[i].pte); + sizeof(u64), u64, ptes[idx].pte); else - ptr[i] = ptes[i].pte; + ptr[i] = ptes[idx].pte; } } @@ -1077,7 +1098,7 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma) xe_pt_commit_prepare_locks_assert(vma); if (xe_vma_is_userptr(vma)) - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); } static void xe_pt_commit(struct xe_vma *vma, @@ -1398,6 +1419,38 @@ static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update) } #if IS_ENABLED(CONFIG_DRM_GPUSVM) +/* + * Acquire/release the svm notifier_lock around xe_pt_svm_userptr_pre_commit() + * and the matching late release in xe_pt_update_ops_run(). Read mode by + * default; write mode when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on, + * because a userptr op in this critical section may invoke the injected + * xe_vma_userptr_force_invalidate() path that calls + * drm_gpusvm_unmap_pages() with ctx->in_notifier=true, which requires the + * lock held for write. + */ +static void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm) +{ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) + down_write(&vm->svm.gpusvm.notifier_lock); +#else + xe_svm_notifier_lock(vm); +#endif +} + +static void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm) +{ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) + up_write(&vm->svm.gpusvm.notifier_lock); +#else + xe_svm_notifier_unlock(vm); +#endif +} +#else +static inline void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm) { } +static inline void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm) { } +#endif + +#if IS_ENABLED(CONFIG_DRM_GPUSVM) #ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma) @@ -1428,7 +1481,7 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma, struct xe_userptr_vma *uvma; unsigned long notifier_seq; - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); if (!xe_vma_is_userptr(vma)) return 0; @@ -1458,7 +1511,7 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op, { int err = 0; - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); switch (op->base.op) { case DRM_GPUVA_OP_MAP: @@ -1530,12 +1583,12 @@ static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) if (err) return err; - xe_svm_notifier_lock(vm); + xe_pt_svm_userptr_notifier_lock(vm); list_for_each_entry(op, &vops->list, link) { err = op_check_svm_userptr(vm, op, pt_update_ops); if (err) { - xe_svm_notifier_unlock(vm); + xe_pt_svm_userptr_notifier_unlock(vm); break; } } @@ -1601,23 +1654,21 @@ static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, return false; } -/* page_size = 2^(reclamation_size + XE_PTE_SHIFT) */ -#define COMPUTE_RECLAIM_ADDRESS_MASK(page_size) \ -({ \ - BUILD_BUG_ON(!__builtin_constant_p(page_size)); \ - ilog2(page_size) - XE_PTE_SHIFT; \ -}) - static int generate_reclaim_entry(struct xe_tile *tile, struct xe_page_reclaim_list *prl, u64 pte, struct xe_pt *xe_child) { struct xe_gt *gt = tile->primary_gt; struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries; - u64 phys_addr = pte & XE_PTE_ADDR_MASK; + bool is_2m = xe_child->level == 1 && (pte & XE_PDE_PS_2M); + bool is_64k = xe_child->level == 0 && ((pte & XE_PTE_PS64) || xe_child->is_compact); + u32 page_shift = is_2m ? ilog2(SZ_2M) : is_64k ? ilog2(SZ_64K) : ilog2(SZ_4K); + /* Physical address bits start at page shift: 2M->[51:21], 64K->[51:16], 4K->[51:12] */ + u64 phys_addr = pte & XE_PAGE_ADDR_MASK(page_shift); + /* Page address is relative to 4K page regardless of entry level */ u64 phys_page = phys_addr >> XE_PTE_SHIFT; int num_entries = prl->num_entries; - u32 reclamation_size; + u32 reclamation_size = page_shift - XE_PTE_SHIFT; xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL); xe_tile_assert(tile, reclaim_entries); @@ -1632,18 +1683,12 @@ static int generate_reclaim_entry(struct xe_tile *tile, * Page size is computed as 2^(reclamation_size + XE_PTE_SHIFT) bytes. * Only 4K, 64K (level 0), and 2M pages are supported by hardware for page reclaim */ - if (xe_child->level == 0 && !(pte & XE_PTE_PS64)) { - xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT, 1); - reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_4K); /* reclamation_size = 0 */ - xe_tile_assert(tile, phys_addr % SZ_4K == 0); - } else if (xe_child->level == 0) { - xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT, 1); - reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_64K); /* reclamation_size = 4 */ - xe_tile_assert(tile, phys_addr % SZ_64K == 0); - } else if (xe_child->level == 1 && pte & XE_PDE_PS_2M) { + if (is_2m) { xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT, 1); - reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_2M); /* reclamation_size = 9 */ - xe_tile_assert(tile, phys_addr % SZ_2M == 0); + } else if (is_64k) { + xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT, 1); + } else if (xe_child->level == 0) { + xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT, 1); } else { xe_page_reclaim_list_abort(tile->primary_gt, prl, "unsupported PTE level=%u pte=%#llx", @@ -1664,6 +1709,48 @@ static int generate_reclaim_entry(struct xe_tile *tile, return 0; } +static int add_pte_to_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl, + struct xe_pt *xe_child, u64 pte, u64 addr) +{ + /* + * In rare scenarios, pte may not be written yet due to racy conditions. + * In such cases, invalidate the PRL and fallback to full PPC invalidation. + */ + if (!pte) { + xe_page_reclaim_list_abort(tile->primary_gt, prl, + "found zero pte at addr=%#llx", addr); + return -EINVAL; + } + + /* Ensure it is a defined page */ + xe_tile_assert(tile, xe_child->level == 0 || + (pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G))); + + /* Account for NULL terminated entry on end (-1) */ + if (prl->num_entries >= XE_PAGE_RECLAIM_MAX_ENTRIES - 1) { + xe_page_reclaim_list_abort(tile->primary_gt, prl, + "overflow while adding pte=%#llx", pte); + return -ENOSPC; + } + + return generate_reclaim_entry(tile, prl, pte, xe_child); +} + +static bool add_compact_pt_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl, + struct xe_device *xe, struct xe_pt *compact_pt, u64 addr) +{ + struct iosys_map *map = &compact_pt->bo->vmap; + + for (pgoff_t i = 0; i < SZ_2M / SZ_64K && xe_page_reclaim_list_valid(prl); i++) { + u64 pte = xe_map_rd(xe, map, i * sizeof(u64), u64); + + if (add_pte_to_prl(tile, prl, compact_pt, pte, addr + i * SZ_64K)) + break; + } + + return xe_page_reclaim_list_valid(prl); +} + static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, unsigned int level, u64 addr, u64 next, struct xe_ptw **child, @@ -1673,21 +1760,22 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); struct xe_pt_stage_unbind_walk *xe_walk = container_of(walk, typeof(*xe_walk), base); - struct xe_device *xe = tile_to_xe(xe_walk->tile); + struct xe_page_reclaim_list *prl = xe_walk->prl; + struct xe_tile *tile = xe_walk->tile; + struct xe_device *xe = tile_to_xe(tile); pgoff_t first = xe_pt_offset(addr, xe_child->level, walk); bool killed; XE_WARN_ON(!*child); XE_WARN_ON(!level); /* Check for leaf node */ - if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) && + if (prl && xe_page_reclaim_list_valid(prl) && xe_child->level <= MAX_HUGEPTE_LEVEL) { struct iosys_map *leaf_map = &xe_child->bo->vmap; pgoff_t count = xe_pt_num_entries(addr, next, xe_child->level, walk); for (pgoff_t i = 0; i < count; i++) { u64 pte; - int ret; /* * If not a leaf pt, skip unless non-leaf pt is interleaved between @@ -1697,10 +1785,23 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, u64 pt_size = 1ULL << walk->shifts[xe_child->level]; bool edge_pt = (i == 0 && !IS_ALIGNED(addr, pt_size)) || (i == count - 1 && !IS_ALIGNED(next, pt_size)); - - if (!edge_pt) { - xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, - xe_walk->prl, + struct xe_pt *child_pt = + container_of(xe_child->base.children[first + i], + struct xe_pt, base); + + /* Compact PTs always fill a full 2M-aligned slot, never an edge. */ + XE_WARN_ON(child_pt->is_compact && edge_pt); + if (edge_pt) + continue; + + /* Walker never descends into compact PTs, descend now */ + if (child_pt->is_compact) { + if (!add_compact_pt_prl(tile, prl, xe, child_pt, + addr + (u64)i * pt_size)) + break; + } else { + xe_page_reclaim_list_abort(tile->primary_gt, + prl, "PT is skipped by walk at level=%u offset=%lu", xe_child->level, first + i); break; @@ -1710,37 +1811,12 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64); - /* - * In rare scenarios, pte may not be written yet due to racy conditions. - * In such cases, invalidate the PRL and fallback to full PPC invalidation. - */ - if (!pte) { - xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, xe_walk->prl, - "found zero pte at addr=%#llx", addr); + if (add_pte_to_prl(tile, prl, xe_child, pte, addr)) break; - } - - /* Ensure it is a defined page */ - xe_tile_assert(xe_walk->tile, xe_child->level == 0 || - (pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G))); /* An entry should be added for 64KB but contigious 4K have XE_PTE_PS64 */ if (pte & XE_PTE_PS64) i += 15; /* Skip other 15 consecutive 4K pages in the 64K page */ - - /* Account for NULL terminated entry on end (-1) */ - if (xe_walk->prl->num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1) { - ret = generate_reclaim_entry(xe_walk->tile, xe_walk->prl, - pte, xe_child); - if (ret) - break; - } else { - /* overflow, mark as invalid */ - xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, xe_walk->prl, - "overflow while adding pte=%#llx", - pte); - break; - } } } @@ -1750,7 +1826,7 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, * Verify if any PTE are potentially dropped at non-leaf levels, either from being * killed or the page walk covers the region. */ - if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) && + if (prl && xe_page_reclaim_list_valid(prl) && xe_child->level > MAX_HUGEPTE_LEVEL && xe_child->num_live) { bool covered = xe_pt_covers(addr, next, xe_child->level, &xe_walk->base); @@ -1759,7 +1835,7 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, * we need to invalidate the PRL. */ if (killed || covered) - xe_page_reclaim_list_abort(xe_walk->tile->primary_gt, xe_walk->prl, + xe_page_reclaim_list_abort(tile->primary_gt, prl, "kill at level=%u addr=%#llx next=%#llx num_live=%u", level, addr, next, xe_child->num_live); } @@ -2289,8 +2365,11 @@ static void xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) { init_llist_head(&pt_update_ops->deferred); + pt_update_ops->current_op = 0; pt_update_ops->start = ~0x0ull; pt_update_ops->last = 0x0ull; + pt_update_ops->needs_svm_lock = false; + pt_update_ops->needs_invalidation = false; xe_page_reclaim_list_init(&pt_update_ops->prl); } @@ -2371,7 +2450,7 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, vma->tile_invalidated & ~BIT(tile->id)); vma->tile_staged &= ~BIT(tile->id); if (xe_vma_is_userptr(vma)) { - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); to_userptr_vma(vma)->userptr.initial_bind = true; } @@ -2407,7 +2486,7 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, if (!vma->tile_present) { list_del_init(&vma->combined_links.rebind); if (xe_vma_is_userptr(vma)) { - xe_svm_assert_held_read(vm); + xe_svm_assert_held_read_or_inject_write(vm); spin_lock(&vm->userptr.invalidated_lock); list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link); @@ -2683,7 +2762,7 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) } if (pt_update_ops->needs_svm_lock) - xe_svm_notifier_unlock(vm); + xe_pt_svm_userptr_notifier_unlock(vm); /* * The last fence is only used for zero bind queue idling; migrate diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h index 84b51d3762a4..a7d1bb708b69 100644 --- a/drivers/gpu/drm/xe/xe_pt_types.h +++ b/drivers/gpu/drm/xe/xe_pt_types.h @@ -34,7 +34,7 @@ struct xe_pt { bool rebind; bool is_compact; #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) - /** addr: Virtual address start address of the PT. */ + /** @addr: Virtual address start address of the PT. */ u64 addr; #endif }; diff --git a/drivers/gpu/drm/xe/xe_pt_walk.h b/drivers/gpu/drm/xe/xe_pt_walk.h index 5c02c244f7de..e91995fa703b 100644 --- a/drivers/gpu/drm/xe/xe_pt_walk.h +++ b/drivers/gpu/drm/xe/xe_pt_walk.h @@ -2,8 +2,8 @@ /* * Copyright © 2022 Intel Corporation */ -#ifndef __XE_PT_WALK__ -#define __XE_PT_WALK__ +#ifndef _XE_PT_WALK_H_ +#define _XE_PT_WALK_H_ #include <linux/pagewalk.h> #include <linux/types.h> @@ -34,7 +34,7 @@ struct xe_pt_walk { * changed during the walk. */ const u64 *shifts; - /** @max_level: Highest populated level in @sizes */ + /** @max_level: Highest populated level in @shifts */ unsigned int max_level; /** * @shared_pt_mode: Whether to skip all entries that are private @@ -49,7 +49,7 @@ struct xe_pt_walk { /** * typedef xe_pt_entry_fn - gpu page-table-walk callback-function - * @parent: The parent page.table. + * @parent: The parent page table. * @offset: The offset (number of entries) into the page table. * @level: The level of @parent. * @addr: The virtual address. @@ -111,14 +111,14 @@ static inline bool xe_pt_covers(u64 addr, u64 end, unsigned int level, } /** - * xe_pt_num_entries: Number of page-table entries of a given range at this + * xe_pt_num_entries - Number of page-table entries of a given range at this * level * @addr: Start address. * @end: End address. * @level: Page table level. * @walk: Walk info. * - * Return: The number of page table entries at this level between @start and + * Return: The number of page table entries at this level between @addr and * @end. */ static inline pgoff_t @@ -132,7 +132,7 @@ xe_pt_num_entries(u64 addr, u64 end, unsigned int level, } /** - * xe_pt_offset: Offset of the page-table entry for a given address. + * xe_pt_offset - Offset of the page-table entry for a given address. * @addr: The address. * @level: Page table level. * @walk: Walk info. diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index 7244090b0782..968b7e70b3f9 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -13,10 +13,12 @@ #include "xe_device_types.h" #include "xe_exec_queue.h" #include "xe_force_wake.h" +#include "xe_guc_exec_queue_types.h" #include "xe_guc_submit.h" #include "xe_gsc_proxy.h" #include "xe_gt_types.h" #include "xe_huc.h" +#include "xe_hw_engine.h" #include "xe_mmio.h" #include "xe_pm.h" #include "xe_pxp_submit.h" @@ -740,6 +742,10 @@ static void pxp_invalidate_queues(struct xe_pxp *pxp) spin_unlock_irq(&pxp->queues.lock); list_for_each_entry_safe(q, tmp, &to_clean, pxp.link) { + drm_dbg(&pxp->xe->drm, + "Killing queue due to PXP termination: eclass=%s, guc_id=%d\n", + xe_hw_engine_class_to_str(q->class), q->guc->id); + xe_exec_queue_kill(q); /* diff --git a/drivers/gpu/drm/xe/xe_pxp.h b/drivers/gpu/drm/xe/xe_pxp.h index 71a23280b900..4fb6e0afffd2 100644 --- a/drivers/gpu/drm/xe/xe_pxp.h +++ b/drivers/gpu/drm/xe/xe_pxp.h @@ -3,8 +3,8 @@ * Copyright(c) 2024, Intel Corporation. All rights reserved. */ -#ifndef __XE_PXP_H__ -#define __XE_PXP_H__ +#ifndef _XE_PXP_H_ +#define _XE_PXP_H_ #include <linux/types.h> @@ -32,4 +32,4 @@ int xe_pxp_key_assign(struct xe_pxp *pxp, struct xe_bo *bo); int xe_pxp_bo_key_check(struct xe_pxp *pxp, struct xe_bo *bo); int xe_pxp_obj_key_check(struct drm_gem_object *obj); -#endif /* __XE_PXP_H__ */ +#endif /* _XE_PXP_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_debugfs.h b/drivers/gpu/drm/xe/xe_pxp_debugfs.h index 988466aad50b..2997de0c90b2 100644 --- a/drivers/gpu/drm/xe/xe_pxp_debugfs.h +++ b/drivers/gpu/drm/xe/xe_pxp_debugfs.h @@ -3,11 +3,11 @@ * Copyright © 2024 Intel Corporation */ -#ifndef __XE_PXP_DEBUGFS_H__ -#define __XE_PXP_DEBUGFS_H__ +#ifndef _XE_PXP_DEBUGFS_H_ +#define _XE_PXP_DEBUGFS_H_ struct xe_pxp; void xe_pxp_debugfs_register(struct xe_pxp *pxp); -#endif /* __XE_PXP_DEBUGFS_H__ */ +#endif /* _XE_PXP_DEBUGFS_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_submit.h b/drivers/gpu/drm/xe/xe_pxp_submit.h index c9efda02f4b0..dbbbe6b92bb2 100644 --- a/drivers/gpu/drm/xe/xe_pxp_submit.h +++ b/drivers/gpu/drm/xe/xe_pxp_submit.h @@ -3,8 +3,8 @@ * Copyright(c) 2024, Intel Corporation. All rights reserved. */ -#ifndef __XE_PXP_SUBMIT_H__ -#define __XE_PXP_SUBMIT_H__ +#ifndef _XE_PXP_SUBMIT_H_ +#define _XE_PXP_SUBMIT_H_ #include <linux/types.h> @@ -19,4 +19,4 @@ int xe_pxp_submit_session_termination(struct xe_pxp *pxp, u32 id); int xe_pxp_submit_session_invalidation(struct xe_pxp_gsc_client_resources *gsc_res, u32 id); -#endif /* __XE_PXP_SUBMIT_H__ */ +#endif /* _XE_PXP_SUBMIT_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_types.h b/drivers/gpu/drm/xe/xe_pxp_types.h index 53e9d48d10fb..ec86306e16f4 100644 --- a/drivers/gpu/drm/xe/xe_pxp_types.h +++ b/drivers/gpu/drm/xe/xe_pxp_types.h @@ -3,8 +3,8 @@ * Copyright(c) 2024, Intel Corporation. All rights reserved. */ -#ifndef __XE_PXP_TYPES_H__ -#define __XE_PXP_TYPES_H__ +#ifndef _XE_PXP_TYPES_H_ +#define _XE_PXP_TYPES_H_ #include <linux/completion.h> #include <linux/iosys-map.h> @@ -132,4 +132,4 @@ struct xe_pxp { u32 last_suspend_key_instance; }; -#endif /* __XE_PXP_TYPES_H__ */ +#endif /* _XE_PXP_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index d84d6a422c45..8c7d54498f38 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -231,6 +231,9 @@ static size_t calc_mem_regions_size(struct xe_device *xe) u32 num_managers = 1; int i; + if (xe_device_is_admin_only(xe)) + return sizeof(struct drm_xe_query_mem_regions); + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) if (ttm_manager_type(&xe->ttm, i)) num_managers++; @@ -259,6 +262,9 @@ static int query_mem_regions(struct xe_device *xe, if (XE_IOCTL_DBG(xe, !mem_regions)) return -ENOMEM; + if (xe_device_is_admin_only(xe)) + goto user_copy; + man = ttm_manager_type(&xe->ttm, XE_PL_TT); mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; /* @@ -297,6 +303,7 @@ static int query_mem_regions(struct xe_device *xe, } } +user_copy: if (!copy_to_user(query_ptr, mem_regions, size)) ret = 0; else diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c new file mode 100644 index 000000000000..4cb16b419b0c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_printk.h" +#include "xe_ras.h" +#include "xe_ras_types.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_event_types.h" + +/* Severity of detected errors */ +enum xe_ras_severity { + XE_RAS_SEV_NOT_SUPPORTED = 0, + XE_RAS_SEV_CORRECTABLE, + XE_RAS_SEV_UNCORRECTABLE, + XE_RAS_SEV_INFORMATIONAL, + XE_RAS_SEV_MAX +}; + +/* Major IP blocks/components where errors can originate */ +enum xe_ras_component { + XE_RAS_COMP_NOT_SUPPORTED = 0, + XE_RAS_COMP_DEVICE_MEMORY, + XE_RAS_COMP_CORE_COMPUTE, + XE_RAS_COMP_RESERVED, + XE_RAS_COMP_PCIE, + XE_RAS_COMP_FABRIC, + XE_RAS_COMP_SOC_INTERNAL, + XE_RAS_COMP_MAX +}; + +static const char *const xe_ras_severities[] = { + [XE_RAS_SEV_NOT_SUPPORTED] = "Not Supported", + [XE_RAS_SEV_CORRECTABLE] = "Correctable Error", + [XE_RAS_SEV_UNCORRECTABLE] = "Uncorrectable Error", + [XE_RAS_SEV_INFORMATIONAL] = "Informational Error", +}; +static_assert(ARRAY_SIZE(xe_ras_severities) == XE_RAS_SEV_MAX); + +static const char *const xe_ras_components[] = { + [XE_RAS_COMP_NOT_SUPPORTED] = "Not Supported", + [XE_RAS_COMP_DEVICE_MEMORY] = "Device Memory", + [XE_RAS_COMP_CORE_COMPUTE] = "Core Compute", + [XE_RAS_COMP_RESERVED] = "Reserved", + [XE_RAS_COMP_PCIE] = "PCIe", + [XE_RAS_COMP_FABRIC] = "Fabric", + [XE_RAS_COMP_SOC_INTERNAL] = "SoC Internal", +}; +static_assert(ARRAY_SIZE(xe_ras_components) == XE_RAS_COMP_MAX); + +static inline const char *sev_to_str(u8 severity) +{ + if (severity >= XE_RAS_SEV_MAX) + severity = XE_RAS_SEV_NOT_SUPPORTED; + + return xe_ras_severities[severity]; +} + +static inline const char *comp_to_str(u8 component) +{ + if (component >= XE_RAS_COMP_MAX) + component = XE_RAS_COMP_NOT_SUPPORTED; + + return xe_ras_components[component]; +} + +void xe_ras_counter_threshold_crossed(struct xe_device *xe, + struct xe_sysctrl_event_response *response) +{ + struct xe_ras_threshold_crossed *pending = (void *)&response->data; + struct xe_ras_error_class *errors = pending->counters; + u32 id, ncounters = pending->ncounters; + + BUILD_BUG_ON(sizeof(response->data) < sizeof(*pending)); + xe_device_assert_mem_access(xe); + + if (!ncounters || ncounters > XE_RAS_NUM_COUNTERS) + xe_err(xe, "sysctrl: unexpected counter threshold crossed %u\n", ncounters); + else + xe_warn(xe, "[RAS]: counter threshold crossed, %u new errors\n", ncounters); + + for (id = 0; id < ncounters && id < XE_RAS_NUM_COUNTERS; id++) { + u8 severity, component; + + severity = errors[id].common.severity; + component = errors[id].common.component; + + xe_warn(xe, "[RAS]: %s %s detected\n", + comp_to_str(component), sev_to_str(severity)); + } +} diff --git a/drivers/gpu/drm/xe/xe_ras.h b/drivers/gpu/drm/xe/xe_ras.h new file mode 100644 index 000000000000..ea90593b62dc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_RAS_H_ +#define _XE_RAS_H_ + +struct xe_device; +struct xe_sysctrl_event_response; + +void xe_ras_counter_threshold_crossed(struct xe_device *xe, + struct xe_sysctrl_event_response *response); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ras_types.h b/drivers/gpu/drm/xe/xe_ras_types.h new file mode 100644 index 000000000000..4e63c67f806a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras_types.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_RAS_TYPES_H_ +#define _XE_RAS_TYPES_H_ + +#include <linux/types.h> + +#define XE_RAS_NUM_COUNTERS 16 + +/** + * struct xe_ras_error_common - Error fields that are common across all products + */ +struct xe_ras_error_common { + /** @severity: Error severity */ + u8 severity; + /** @component: IP block where error originated */ + u8 component; +} __packed; + +/** + * struct xe_ras_error_unit - Error unit information + */ +struct xe_ras_error_unit { + /** @tile: Tile identifier */ + u8 tile; + /** @instance: Instance identifier specific to IP */ + u32 instance; +} __packed; + +/** + * struct xe_ras_error_cause - Error cause information + */ +struct xe_ras_error_cause { + /** @cause: Cause/checker */ + u32 cause; + /** @reserved: For future use */ + u8 reserved; +} __packed; + +/** + * struct xe_ras_error_product - Error fields that are specific to the product + */ +struct xe_ras_error_product { + /** @unit: Unit within IP block */ + struct xe_ras_error_unit unit; + /** @cause: Cause/checker */ + struct xe_ras_error_cause cause; +} __packed; + +/** + * struct xe_ras_error_class - Combines common and product-specific parts + */ +struct xe_ras_error_class { + /** @common: Common error type and component */ + struct xe_ras_error_common common; + /** @product: Product-specific unit and cause */ + struct xe_ras_error_product product; +} __packed; + +/** + * struct xe_ras_threshold_crossed - Data for threshold crossed event + */ +struct xe_ras_threshold_crossed { + /** @ncounters: Number of error counters that crossed thresholds */ + u32 ncounters; + /** @counters: Array of error counters that crossed threshold */ + struct xe_ras_error_class counters[XE_RAS_NUM_COUNTERS]; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c index 2df0277efb2f..e328f5072557 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr.c +++ b/drivers/gpu/drm/xe/xe_reg_sr.c @@ -70,14 +70,49 @@ static void reg_sr_inc_error(struct xe_reg_sr *sr) #endif } +static struct xe_reg sanitize_mcr(struct xe_reg_sr *sr, + const struct xe_reg_sr_entry *e, + struct xe_gt *gt) +{ + struct xe_reg reg = e->reg; + bool is_mcr; + + /* + * We need the gt structure to check MCR ranges. + */ + if (!gt) + return reg; + + is_mcr = xe_gt_mcr_check_reg(gt, reg); + + if (is_mcr && !reg.mcr) { + reg.mcr = 1; + xe_gt_notice(gt, "xe_reg_sr_entry using non-MCR register for address 0x%x, forcing MCR\n", + reg.addr); + reg_sr_inc_error(sr); + } + + if (!is_mcr && reg.mcr) { + reg.mcr = 0; + xe_gt_notice(gt, "xe_reg_sr_entry using MCR register for address 0x%x, forcing non-MCR\n", + reg.addr); + reg_sr_inc_error(sr); + } + + return reg; +} + int xe_reg_sr_add(struct xe_reg_sr *sr, const struct xe_reg_sr_entry *e, struct xe_gt *gt) { unsigned long idx = e->reg.addr; struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); + struct xe_reg reg; int ret; + reg = sanitize_mcr(sr, e, gt); + if (pentry) { if (!compatible_entries(pentry, e)) { ret = -EINVAL; @@ -98,6 +133,7 @@ int xe_reg_sr_add(struct xe_reg_sr *sr, } *pentry = *e; + pentry->reg = reg; ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); if (ret) goto fail_free; diff --git a/drivers/gpu/drm/xe/xe_reg_sr.h b/drivers/gpu/drm/xe/xe_reg_sr.h index 1ec6e8ecf278..d26cf4713383 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr.h +++ b/drivers/gpu/drm/xe/xe_reg_sr.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_REG_SR_ -#define _XE_REG_SR_ +#ifndef _XE_REG_SR_H_ +#define _XE_REG_SR_H_ /* * Reg save/restore bookkeeping diff --git a/drivers/gpu/drm/xe/xe_reg_sr_types.h b/drivers/gpu/drm/xe/xe_reg_sr_types.h index ebe11f237fa2..0a6695db2967 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr_types.h +++ b/drivers/gpu/drm/xe/xe_reg_sr_types.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_REG_SR_TYPES_ -#define _XE_REG_SR_TYPES_ +#ifndef _XE_REG_SR_TYPES_H_ +#define _XE_REG_SR_TYPES_H_ #include <linux/types.h> #include <linux/xarray.h> diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c index 8cc313182968..526907d2d824 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.c +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -9,6 +9,7 @@ #include "regs/xe_gt_regs.h" #include "regs/xe_oa_regs.h" #include "xe_device.h" +#include "xe_gt.h" #include "xe_gt_types.h" #include "xe_gt_printk.h" #include "xe_platform_types.h" @@ -33,7 +34,14 @@ static bool match_has_mert(const struct xe_device *xe, return xe_device_has_mert((struct xe_device *)xe); } -static const struct xe_rtp_entry_sr register_whitelist[] = { +static bool match_multi_queue_class(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_gt_supports_multi_queue(gt, hwe->class); +} + +static const struct xe_rtp_table_sr register_whitelist = XE_RTP_TABLE_SR( { XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(WHITELIST(PS_INVOCATION_COUNT, @@ -54,6 +62,12 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { RING_FORCE_TO_NONPRIV_ACCESS_RD, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, + { XE_RTP_NAME("allow_read_queue_timestamp"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, 3511), FUNC(match_multi_queue_class)), + XE_RTP_ACTIONS(WHITELIST(RING_QUEUE_TIMESTAMP(0), + RING_FORCE_TO_NONPRIV_ACCESS_RD, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, { XE_RTP_NAME("16014440446"), XE_RTP_RULES(PLATFORM(PVC)), XE_RTP_ACTIONS(WHITELIST(XE_REG(0x4400), @@ -89,11 +103,16 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { WHITELIST(VFLSKPD, RING_FORCE_TO_NONPRIV_ACCESS_RW)) }, +); + +static const struct xe_rtp_table_sr oa_whitelist = XE_RTP_TABLE_SR( + +#define WHITELIST_DENY(r, f) WHITELIST(r, (f) | RING_FORCE_TO_NONPRIV_DENY) #define WHITELIST_OA_MMIO_TRG(trg, status, head) \ - WHITELIST(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \ - WHITELIST(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \ - WHITELIST(head, RING_FORCE_TO_NONPRIV_ACCESS_RD | RING_FORCE_TO_NONPRIV_RANGE_4) + WHITELIST_DENY(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \ + WHITELIST_DENY(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \ + WHITELIST_DENY(head, RING_FORCE_TO_NONPRIV_ACCESS_RD | RING_FORCE_TO_NONPRIV_RANGE_4) #define WHITELIST_OAG_MMIO_TRG \ WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR) @@ -110,7 +129,7 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ)) #define WHITELIST_OA_MERT_MMIO_TRG \ - WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_HEAD_POINTER) + WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_TAIL_POINTER) { XE_RTP_NAME("oag_mmio_trg_rcs"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), @@ -140,11 +159,12 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COPY)), XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG) }, -}; +); -static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) +static int whitelist_apply_to_hwe(struct xe_hw_engine *hwe, struct xe_reg_sr *in, + struct xe_reg_sr *out, int first_slot) { - struct xe_reg_sr *sr = &hwe->reg_whitelist; + struct xe_reg_sr *sr = in; struct xe_reg_sr_entry *entry; struct drm_printer p; unsigned long reg; @@ -153,7 +173,7 @@ static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) xe_gt_dbg(hwe->gt, "Add %s whitelist to engine\n", sr->name); p = xe_gt_dbg_printer(hwe->gt); - slot = 0; + slot = first_slot; xa_for_each(&sr->xa, reg, entry) { struct xe_reg_sr_entry hwe_entry = { .reg = RING_FORCE_TO_NONPRIV(hwe->mmio_base, slot), @@ -170,10 +190,12 @@ static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) } xe_reg_whitelist_print_entry(&p, 0, reg, entry); - xe_reg_sr_add(&hwe->reg_sr, &hwe_entry, hwe->gt); + xe_reg_sr_add(out, &hwe_entry, hwe->gt); slot++; } + + return slot; } /** @@ -187,10 +209,78 @@ static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe) { struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + int first_oa_slot; + + xe_rtp_process_to_sr(&ctx, ®ister_whitelist, &hwe->reg_whitelist, false); + first_oa_slot = whitelist_apply_to_hwe(hwe, &hwe->reg_whitelist, &hwe->reg_sr, 0); + + xe_rtp_process_to_sr(&ctx, &oa_whitelist, &hwe->oa_whitelist, false); + + /* + * Save oa nonpriv registers to hwe->oa_sr, from which oa registers are whitelisted + * or de-whitelisted, by toggling the 'deny' bit on oa stream open/close + */ + whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->oa_sr, first_oa_slot); + + /* + * Also save oa nonpriv registers to hwe->reg_sr, to ensure oa registers are not + * whitelisted by default after probe, gt reset, resume and engine reset + */ + whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->reg_sr, first_oa_slot); +} + +static void __whitelist_oa_regs(struct xe_hw_engine *hwe, bool whitelist) +{ + struct xe_reg_sr_entry *entry; + unsigned long reg; + + xa_for_each(&hwe->oa_sr.xa, reg, entry) { + if (whitelist) + entry->set_bits &= ~RING_FORCE_TO_NONPRIV_DENY; + else + entry->set_bits |= RING_FORCE_TO_NONPRIV_DENY; + } + + xe_reg_sr_apply_mmio(&hwe->oa_sr, hwe->gt); +} + +/** + * xe_reg_whitelist_oa_regs - whitelist oa registers for gt + * @gt: gt to whitelist oa registers for + * + * Whitelist OA registers by resetting RING_FORCE_TO_NONPRIV_DENY + */ +void xe_reg_whitelist_oa_regs(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + lockdep_assert_held(>->oa.gt_lock); + if (gt->oa.whitelist_count++) + return; + + for_each_hw_engine(hwe, gt, id) + __whitelist_oa_regs(hwe, true); +} + +/** + * xe_reg_dewhitelist_oa_regs - dewhitelist oa registers for gt + * @gt: gt to dewhitelist oa registers for + * + * Dewhitelist OA registers by setting RING_FORCE_TO_NONPRIV_DENY + */ +void xe_reg_dewhitelist_oa_regs(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + lockdep_assert_held(>->oa.gt_lock); + xe_assert(gt_to_xe(gt), gt->oa.whitelist_count); + if (--gt->oa.whitelist_count) + return; - xe_rtp_process_to_sr(&ctx, register_whitelist, ARRAY_SIZE(register_whitelist), - &hwe->reg_whitelist, false); - whitelist_apply_to_hwe(hwe); + for_each_hw_engine(hwe, gt, id) + __whitelist_oa_regs(hwe, false); } /** diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.h b/drivers/gpu/drm/xe/xe_reg_whitelist.h index 69b121d377da..e1eb1b7d5480 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.h +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.h @@ -3,18 +3,22 @@ * Copyright © 2023 Intel Corporation */ -#ifndef _XE_REG_WHITELIST_ -#define _XE_REG_WHITELIST_ +#ifndef _XE_REG_WHITELIST_H_ +#define _XE_REG_WHITELIST_H_ #include <linux/types.h> struct drm_printer; +struct xe_gt; struct xe_hw_engine; struct xe_reg_sr; struct xe_reg_sr_entry; void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe); +void xe_reg_whitelist_oa_regs(struct xe_gt *gt); +void xe_reg_dewhitelist_oa_regs(struct xe_gt *gt); + void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent, u32 reg, struct xe_reg_sr_entry *entry); diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h index 5f4ab08c0686..0522caafd89d 100644 --- a/drivers/gpu/drm/xe/xe_res_cursor.h +++ b/drivers/gpu/drm/xe/xe_res_cursor.h @@ -101,7 +101,15 @@ static inline void xe_res_first(struct ttm_resource *res, cur->mem_type = res->mem_type; switch (cur->mem_type) { - case XE_PL_STOLEN: + case XE_PL_STOLEN: { + /* res->start is in pages (ttm_range_manager). */ + cur->start = (res->start << PAGE_SHIFT) + start; + cur->size = size; + cur->remaining = size; + cur->node = NULL; + cur->mm = NULL; + break; + } case XE_PL_VRAM0: case XE_PL_VRAM1: { struct gpu_buddy_block *block; @@ -289,6 +297,10 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size) switch (cur->mem_type) { case XE_PL_STOLEN: + /* Just advance within the contiguous region. */ + cur->start += size; + cur->size = cur->remaining; + break; case XE_PL_VRAM0: case XE_PL_VRAM1: start = size - cur->size; diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index cfeb4fc7d217..39a670e91ba7 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -269,8 +269,12 @@ static u32 get_ppgtt_flag(struct xe_sched_job *job) static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, u32 *dw, int i) { + const struct xe_reg reg = xe_lrc_is_multi_queue(lrc) ? + RING_QUEUE_TIMESTAMP(0) : + RING_CTX_TIMESTAMP(0); + dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; - dw[i++] = RING_CTX_TIMESTAMP(0).addr; + dw[i++] = reg.addr; dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); dw[i++] = 0; @@ -281,7 +285,7 @@ static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, if (IS_SRIOV_VF(xe)) { dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; - dw[i++] = RING_CTX_TIMESTAMP(0).addr; + dw[i++] = reg.addr; dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc); dw[i++] = 0; } diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c index 991f218f1cc3..83a40e1f9528 100644 --- a/drivers/gpu/drm/xe/xe_rtp.c +++ b/drivers/gpu/drm/xe/xe_rtp.c @@ -10,6 +10,7 @@ #include <uapi/drm/xe_drm.h> #include "xe_configfs.h" +#include "xe_device.h" #include "xe_gt.h" #include "xe_gt_topology.h" #include "xe_reg_sr.h" @@ -29,146 +30,199 @@ static bool has_samedia(const struct xe_device *xe) return xe->info.media_verx100 >= 1300; } -static bool rule_matches(const struct xe_device *xe, - struct xe_gt *gt, - struct xe_hw_engine *hwe, - const struct xe_rtp_rule *rules, - unsigned int n_rules) +struct rule_match_ctx { + const struct xe_device *xe; + struct xe_gt *gt; + struct xe_hw_engine *hwe; + const struct xe_rtp_rule *rules; + const unsigned int n_rules; + unsigned int head; + int err; +}; + +static bool rule_is_item(const struct xe_rtp_rule *r) +{ + return r->match_type != XE_RTP_MATCH_OR; +} + +static bool rule_match_item(struct rule_match_ctx *match_ctx) +{ + const struct xe_device *xe = match_ctx->xe; + struct xe_gt *gt = match_ctx->gt; + struct xe_hw_engine *hwe = match_ctx->hwe; + const struct xe_rtp_rule *r = &match_ctx->rules[match_ctx->head]; + + switch (r->match_type) { + case XE_RTP_MATCH_PLATFORM: + return xe->info.platform == r->platform; + case XE_RTP_MATCH_SUBPLATFORM: + return xe->info.platform == r->platform && + xe->info.subplatform == r->subplatform; + case XE_RTP_MATCH_PLATFORM_STEP: + if (drm_WARN_ON(&xe->drm, xe->info.step.platform == STEP_NONE)) + return false; + + return xe->info.step.platform >= r->step_start && + xe->info.step.platform < r->step_end; + case XE_RTP_MATCH_GRAPHICS_VERSION: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.graphics_verx100 == r->ver_start && + (!has_samedia(xe) || !xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_GRAPHICS_VERSION_RANGE: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.graphics_verx100 >= r->ver_start && + xe->info.graphics_verx100 <= r->ver_end && + (!has_samedia(xe) || !xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.graphics_verx100 == r->ver_start; + case XE_RTP_MATCH_GRAPHICS_STEP: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.step.graphics >= r->step_start && + xe->info.step.graphics < r->step_end && + (!has_samedia(xe) || !xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_MEDIA_VERSION: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.media_verx100 == r->ver_start && + (!has_samedia(xe) || xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_MEDIA_VERSION_RANGE: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.media_verx100 >= r->ver_start && + xe->info.media_verx100 <= r->ver_end && + (!has_samedia(xe) || xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_MEDIA_STEP: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.step.media >= r->step_start && + xe->info.step.media < r->step_end && + (!has_samedia(xe) || xe_gt_is_media_type(gt)); + case XE_RTP_MATCH_MEDIA_VERSION_ANY_GT: + if (drm_WARN_ON(&xe->drm, !gt)) + return false; + + return xe->info.media_verx100 == r->ver_start; + case XE_RTP_MATCH_INTEGRATED: + return !xe->info.is_dgfx; + case XE_RTP_MATCH_DISCRETE: + return xe->info.is_dgfx; + case XE_RTP_MATCH_ENGINE_CLASS: + if (drm_WARN_ON(&xe->drm, !hwe)) + return false; + + return hwe->class == r->engine_class; + case XE_RTP_MATCH_NOT_ENGINE_CLASS: + if (drm_WARN_ON(&xe->drm, !hwe)) + return false; + + return hwe->class != r->engine_class; + case XE_RTP_MATCH_FUNC: + return r->match_func(xe, gt, hwe); + default: + drm_warn(&xe->drm, "Invalid RTP match %u\n", + r->match_type); + return false; + } +} + +/* + * Match a conjunctive set of rules (rules joined by an implicit "AND"). + * + * Once one item evaluates to false, the remaining items are not evaluated + * anymore. Nevetheless, all rules are consumed to allow detecting syntax + * errors. + */ +static bool rule_match_and(struct rule_match_ctx *match_ctx, bool parse_only) { - const struct xe_rtp_rule *r; - unsigned int i, rcount = 0; - bool match; - - for (r = rules, i = 0; i < n_rules; r = &rules[++i]) { - switch (r->match_type) { - case XE_RTP_MATCH_OR: - /* - * This is only reached if a complete set of - * rules passed or none were evaluated. For both cases, - * shortcut the other rules and return the proper value. - */ - goto done; - case XE_RTP_MATCH_PLATFORM: - match = xe->info.platform == r->platform; - break; - case XE_RTP_MATCH_SUBPLATFORM: - match = xe->info.platform == r->platform && - xe->info.subplatform == r->subplatform; - break; - case XE_RTP_MATCH_PLATFORM_STEP: - if (drm_WARN_ON(&xe->drm, xe->info.step.platform == STEP_NONE)) - return false; - - match = xe->info.step.platform >= r->step_start && - xe->info.step.platform < r->step_end; - break; - case XE_RTP_MATCH_GRAPHICS_VERSION: - if (drm_WARN_ON(&xe->drm, !gt)) - return false; - - match = xe->info.graphics_verx100 == r->ver_start && - (!has_samedia(xe) || !xe_gt_is_media_type(gt)); - break; - case XE_RTP_MATCH_GRAPHICS_VERSION_RANGE: - if (drm_WARN_ON(&xe->drm, !gt)) - return false; - - match = xe->info.graphics_verx100 >= r->ver_start && - xe->info.graphics_verx100 <= r->ver_end && - (!has_samedia(xe) || !xe_gt_is_media_type(gt)); - break; - case XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT: - if (drm_WARN_ON(&xe->drm, !gt)) - return false; - - match = xe->info.graphics_verx100 == r->ver_start; - break; - case XE_RTP_MATCH_GRAPHICS_STEP: - if (drm_WARN_ON(&xe->drm, !gt)) - return false; - - match = xe->info.step.graphics >= r->step_start && - xe->info.step.graphics < r->step_end && - (!has_samedia(xe) || !xe_gt_is_media_type(gt)); - break; - case XE_RTP_MATCH_MEDIA_VERSION: - if (drm_WARN_ON(&xe->drm, !gt)) - return false; - - match = xe->info.media_verx100 == r->ver_start && - (!has_samedia(xe) || xe_gt_is_media_type(gt)); - break; - case XE_RTP_MATCH_MEDIA_VERSION_RANGE: - if (drm_WARN_ON(&xe->drm, !gt)) - return false; - - match = xe->info.media_verx100 >= r->ver_start && - xe->info.media_verx100 <= r->ver_end && - (!has_samedia(xe) || xe_gt_is_media_type(gt)); - break; - case XE_RTP_MATCH_MEDIA_STEP: - if (drm_WARN_ON(&xe->drm, !gt)) - return false; - - match = xe->info.step.media >= r->step_start && - xe->info.step.media < r->step_end && - (!has_samedia(xe) || xe_gt_is_media_type(gt)); - break; - case XE_RTP_MATCH_MEDIA_VERSION_ANY_GT: - if (drm_WARN_ON(&xe->drm, !gt)) - return false; - - match = xe->info.media_verx100 == r->ver_start; - break; - case XE_RTP_MATCH_INTEGRATED: - match = !xe->info.is_dgfx; - break; - case XE_RTP_MATCH_DISCRETE: - match = xe->info.is_dgfx; - break; - case XE_RTP_MATCH_ENGINE_CLASS: - if (drm_WARN_ON(&xe->drm, !hwe)) - return false; - - match = hwe->class == r->engine_class; - break; - case XE_RTP_MATCH_NOT_ENGINE_CLASS: - if (drm_WARN_ON(&xe->drm, !hwe)) - return false; - - match = hwe->class != r->engine_class; - break; - case XE_RTP_MATCH_FUNC: - match = r->match_func(xe, gt, hwe); - break; - default: - drm_warn(&xe->drm, "Invalid RTP match %u\n", - r->match_type); + bool match = true; + unsigned int count = 0; + + while (match_ctx->head < match_ctx->n_rules && + rule_is_item(&match_ctx->rules[match_ctx->head])) { + if (!parse_only) + match = rule_match_item(match_ctx); + + if (!match) + parse_only = true; + + match_ctx->head++; + count++; + } + + if (drm_WARN_ON(&match_ctx->xe->drm, !count)) { + match_ctx->err = -EINVAL; + + if (!parse_only) match = false; - } + } - if (!match) { - /* - * Advance rules until we find XE_RTP_MATCH_OR to check - * if there's another set of conditions to check - */ - while (++i < n_rules && rules[i].match_type != XE_RTP_MATCH_OR) - ; + return match; +} + +/* + * Match a disjunctive set of rules (subset of rules joined by + * "XE_RTP_MATCH_OR"). + * + * Once one subset evaluates to true, the remaining items are not evaluated + * anymore. Nevetheless, all rules are consumed to allow detecting syntax + * errors. + */ +static bool rule_match_or(struct rule_match_ctx *match_ctx) +{ + bool match = rule_match_and(match_ctx, false); - if (i >= n_rules) - return false; + while (match_ctx->head < match_ctx->n_rules && + match_ctx->rules[match_ctx->head].match_type == XE_RTP_MATCH_OR) { + /* Consume XE_RTP_MATCH_OR. */ + match_ctx->head++; - rcount = 0; - } else { - rcount++; - } + match = rule_match_and(match_ctx, match); } -done: - if (drm_WARN_ON(&xe->drm, !rcount)) - return false; + return match; +} - return true; +static bool rule_matches_with_err(const struct xe_device *xe, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_rule *rules, + unsigned int n_rules, + int *err) +{ + struct rule_match_ctx match_ctx = { + .xe = xe, + .gt = gt, + .hwe = hwe, + .rules = rules, + .n_rules = n_rules, + }; + bool match = rule_match_or(&match_ctx); + + if (err) + *err = match_ctx.err; + + return match; +} + +static bool rule_matches(const struct xe_device *xe, + struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_rule *rules, + unsigned int n_rules) +{ + return rule_matches_with_err(xe, gt, hwe, rules, n_rules, NULL); } static void rtp_add_sr_entry(const struct xe_rtp_action *action, @@ -272,8 +326,7 @@ static void rtp_mark_active(struct xe_device *xe, * xe_rtp_process_to_sr - Process all rtp @entries, adding the matching ones to * the save-restore argument. * @ctx: The context for processing the table, with one of device, gt or hwe - * @entries: Table with RTP definitions - * @n_entries: Number of entries to process, usually ARRAY_SIZE(entries) + * @table: Table with RTP definitions * @sr: Save-restore struct where matching rules execute the action. This can be * viewed as the "coalesced view" of multiple the tables. The bits for each * register set are expected not to collide with previously added entries @@ -285,12 +338,10 @@ static void rtp_mark_active(struct xe_device *xe, * used to calculate the right register offset */ void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx, - const struct xe_rtp_entry_sr *entries, - size_t n_entries, + const struct xe_rtp_table_sr *table, struct xe_reg_sr *sr, bool process_in_vf) { - const struct xe_rtp_entry_sr *entry; struct xe_hw_engine *hwe = NULL; struct xe_gt *gt = NULL; struct xe_device *xe = NULL; @@ -300,9 +351,10 @@ void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx, if (!process_in_vf && IS_SRIOV_VF(xe)) return; - xe_assert(xe, entries); + xe_assert(xe, table->entries); - for (entry = entries; entry - entries < n_entries; entry++) { + for (size_t i = 0; i < table->n_entries; i++) { + const struct xe_rtp_entry_sr *entry = &table->entries[i]; bool match = false; if (entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE) { @@ -317,41 +369,51 @@ void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx, } if (match) - rtp_mark_active(xe, ctx, entry - entries); + rtp_mark_active(xe, ctx, i); } } EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process_to_sr); /** - * xe_rtp_process - Process all rtp @entries, without running any action + * xe_rtp_process - Process all entries in rtp @table, without running any action * @ctx: The context for processing the table, with one of device, gt or hwe - * @entries: Table with RTP definitions + * @table: Table with RTP definitions * - * Walk the table pointed by @entries (with an empty sentinel), executing the + * Walk the table pointed by @table, executing the * rules. One difference from xe_rtp_process_to_sr(): there is no action * associated with each entry since this uses struct xe_rtp_entry. Its main use * is for marking active workarounds via * xe_rtp_process_ctx_enable_active_tracking(). */ void xe_rtp_process(struct xe_rtp_process_ctx *ctx, - const struct xe_rtp_entry *entries) + const struct xe_rtp_table *table) { - const struct xe_rtp_entry *entry; struct xe_hw_engine *hwe; struct xe_gt *gt; struct xe_device *xe; rtp_get_context(ctx, &hwe, >, &xe); - for (entry = entries; entry && entry->rules; entry++) { + xe_assert(xe, table->entries); + + for (size_t i = 0; i < table->n_entries; i++) { + const struct xe_rtp_entry *entry = &table->entries[i]; + if (!rule_matches(xe, gt, hwe, entry->rules, entry->n_rules)) continue; - rtp_mark_active(xe, ctx, entry - entries); + rtp_mark_active(xe, ctx, i); } } EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process); +bool xe_rtp_match_always(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return true; +} + bool xe_rtp_match_even_instance(const struct xe_device *xe, const struct xe_gt *gt, const struct xe_hw_engine *hwe) @@ -397,3 +459,14 @@ bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe, { return xe->info.has_flat_ccs; } + +bool xe_rtp_match_has_msix(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_device_has_msix(xe); +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_rtp.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h index 7d6daa7eb1e4..2cc65053cd07 100644 --- a/drivers/gpu/drm/xe/xe_rtp.h +++ b/drivers/gpu/drm/xe/xe_rtp.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_RTP_ -#define _XE_RTP_ +#ifndef _XE_RTP_H_ +#define _XE_RTP_H_ #include <linux/types.h> #include <linux/xarray.h> @@ -394,18 +394,39 @@ struct xe_reg_sr; * XE_RTP_RULES - Helper to set multiple rules to a struct xe_rtp_entry_sr entry * @...: Rules * - * At least one rule is needed and up to 12 are supported. Multiple rules are - * AND'ed together, i.e. all the rules must evaluate to true for the entry to - * be processed. See XE_RTP_MATCH_* for the possible match rules. Example: + * When an RTP table is being processed, the rules of each entry are evaluated + * to check if they match the target entity (platform, gt or hwe, depending on + * the specific RTP table). + * + * The sequence of arguments of this macro must follow the following eBNF + * grammar:: + * + * rules = disjunction; + * disjunction = conjunction, { "OR", conjunction }; + * conjunction = single_rule, { single_rule }; + * (* the AND operator is implicit *) + * single_rule = ? GRAPHICS_VERSION(...), MEDIA_VERSION(...), + * FUNC(...), etc ? + * + * Examples: * * .. code-block:: c * * const struct xe_rtp_entry_sr wa_entries[] = { * ... - * { XE_RTP_NAME("test-entry"), + * { XE_RTP_NAME("entry-a"), + * // Match DG2-G10 with graphics steppings A0 up-to B0 + * // (exclusive). * XE_RTP_RULES(SUBPLATFORM(DG2, G10), GRAPHICS_STEP(A0, B0)), * ... * }, + * { XE_RTP_NAME("entry-b"), + * // Match graphics version 20 (all steppings) or graphics + * // version 30 steppings A0 up-to B0 (exclusive). + * XE_RTP_RULES(GRAPHICS_VERSION(2000), OR, + * GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0)) + * ... + * }, * ... * }; */ @@ -440,6 +461,24 @@ struct xe_reg_sr; XE_RTP_PASTE_FOREACH(ACTION_, COMMA, (__VA_ARGS__)) \ } +/* + * Note: ARRAY_SIZE() cannot be used here because it expands through + * __must_be_array() -> __BUILD_BUG_ON_ZERO_MSG() -> _Static_assert inside + * sizeof(struct{}), which clang < 21 rejects when the compound literal + * contains non-compile-time-constant initializers. + */ +#define XE_RTP_TABLE_SR(...) { \ + .entries = (const struct xe_rtp_entry_sr[]){__VA_ARGS__}, \ + .n_entries = sizeof((const struct xe_rtp_entry_sr[]){__VA_ARGS__}) / \ + sizeof(struct xe_rtp_entry_sr), \ +} + +#define XE_RTP_TABLE(...) { \ + .entries = (const struct xe_rtp_entry[]){__VA_ARGS__}, \ + .n_entries = sizeof((const struct xe_rtp_entry[]){__VA_ARGS__}) / \ + sizeof(struct xe_rtp_entry), \ +} + #define XE_RTP_PROCESS_CTX_INITIALIZER(arg__) _Generic((arg__), \ struct xe_hw_engine * : (struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_ENGINE }, \ struct xe_gt * : (struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_GT }, \ @@ -450,16 +489,28 @@ void xe_rtp_process_ctx_enable_active_tracking(struct xe_rtp_process_ctx *ctx, size_t n_entries); void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx, - const struct xe_rtp_entry_sr *entries, - size_t n_entries, struct xe_reg_sr *sr, + const struct xe_rtp_table_sr *table, + struct xe_reg_sr *sr, bool process_in_vf); void xe_rtp_process(struct xe_rtp_process_ctx *ctx, - const struct xe_rtp_entry *entries); + const struct xe_rtp_table *table); /* Match functions to be used with XE_RTP_MATCH_FUNC */ /** + * xe_rtp_match_always - Match RTP entry unconditionally + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true, regardless of inputs + */ +bool xe_rtp_match_always(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +/** * xe_rtp_match_even_instance - Match if engine instance is even * @xe: Device structure * @gt: GT structure @@ -524,4 +575,16 @@ bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe, const struct xe_gt *gt, const struct xe_hw_engine *hwe); +/** + * xe_rtp_match_has_msix - Match when platform has MSI-X + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true if platform has MSI-X interrupt support + */ +bool xe_rtp_match_has_msix(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + #endif diff --git a/drivers/gpu/drm/xe/xe_rtp_helpers.h b/drivers/gpu/drm/xe/xe_rtp_helpers.h index a33b0ae98bbc..ffa8b2f8828d 100644 --- a/drivers/gpu/drm/xe/xe_rtp_helpers.h +++ b/drivers/gpu/drm/xe/xe_rtp_helpers.h @@ -3,8 +3,8 @@ * Copyright © 2023 Intel Corporation */ -#ifndef _XE_RTP_HELPERS_ -#define _XE_RTP_HELPERS_ +#ifndef _XE_RTP_HELPERS_H_ +#define _XE_RTP_HELPERS_H_ #ifndef _XE_RTP_INCLUDE_PRIVATE_HELPERS #error "This header is supposed to be included by xe_rtp.h only" @@ -66,6 +66,8 @@ #define XE_RTP_PASTE_10(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_9(prefix_, sep_, _XE_TUPLE_TAIL args_) #define XE_RTP_PASTE_11(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_10(prefix_, sep_, _XE_TUPLE_TAIL args_) #define XE_RTP_PASTE_12(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_11(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_13(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_12(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_14(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_13(prefix_, sep_, _XE_TUPLE_TAIL args_) /* * XE_RTP_DROP_CAST - Drop cast to convert a compound statement to a initializer diff --git a/drivers/gpu/drm/xe/xe_rtp_types.h b/drivers/gpu/drm/xe/xe_rtp_types.h index 166251615be1..58018ae4f8cc 100644 --- a/drivers/gpu/drm/xe/xe_rtp_types.h +++ b/drivers/gpu/drm/xe/xe_rtp_types.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_RTP_TYPES_ -#define _XE_RTP_TYPES_ +#ifndef _XE_RTP_TYPES_H_ +#define _XE_RTP_TYPES_H_ #include <linux/types.h> @@ -112,6 +112,16 @@ struct xe_rtp_entry { u8 n_rules; }; +struct xe_rtp_table_sr { + const struct xe_rtp_entry_sr *entries; + size_t n_entries; +}; + +struct xe_rtp_table { + const struct xe_rtp_entry *entries; + size_t n_entries; +}; + enum xe_rtp_process_type { XE_RTP_PROCESS_TYPE_DEVICE, XE_RTP_PROCESS_TYPE_GT, diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c index 47a6e0fd66e0..33bd754d138f 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf.c @@ -20,11 +20,6 @@ #include "xe_sriov_pf_sysfs.h" #include "xe_sriov_printk.h" -static bool wanted_admin_only(struct xe_device *xe) -{ - return xe_configfs_admin_only_pf(to_pci_dev(xe->drm.dev)); -} - static unsigned int wanted_max_vfs(struct xe_device *xe) { return xe_configfs_get_max_vfs(to_pci_dev(xe->drm.dev)); @@ -79,7 +74,6 @@ bool xe_sriov_pf_readiness(struct xe_device *xe) pf_reduce_totalvfs(xe, newlimit); - xe->sriov.pf.admin_only = wanted_admin_only(xe); xe->sriov.pf.device_total_vfs = totalvfs; xe->sriov.pf.driver_max_vfs = newlimit; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h index 0fcc6cec4afc..19f6f8331c8d 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h @@ -7,6 +7,7 @@ #define _XE_SRIOV_PF_HELPERS_H_ #include "xe_assert.h" +#include "xe_device.h" #include "xe_device_types.h" #include "xe_sriov.h" #include "xe_sriov_types.h" @@ -57,7 +58,7 @@ static inline unsigned int xe_sriov_pf_num_vfs(const struct xe_device *xe) static inline bool xe_sriov_pf_admin_only(const struct xe_device *xe) { xe_assert(xe, IS_SRIOV_PF(xe)); - return xe->sriov.pf.admin_only; + return xe_device_is_admin_only(xe); } static inline struct mutex *xe_sriov_pf_master_mutex(struct xe_device *xe) diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c index abe3677d33ed..0ec7ea83f12a 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c @@ -41,6 +41,8 @@ static int pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs) int err; for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_config_set_fair_sched(gt, num_vfs); + result = result ?: err; err = xe_gt_sriov_pf_config_set_fair(gt, VFID(1), num_vfs); result = result ?: err; } @@ -103,6 +105,45 @@ int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs) return 0; } +static int pf_reprovision_default(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, false); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_exec_quantum_locked(gt, PFID, 0); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_preempt_timeout_locked(gt, PFID, 0); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_reprovision_default() - Reprovision default PF in auto-mode. + * @xe: the PF &xe_device + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_reprovision_default(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + if (!pf_auto_provisioning_mode(xe)) + return 0; + + return pf_reprovision_default(xe); +} + /** * xe_sriov_pf_provision_set_mode() - Change VFs provision mode. * @xe: the PF &xe_device diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h index f26f49539697..b15e4d7ad940 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h @@ -30,6 +30,7 @@ int xe_sriov_pf_provision_query_vf_vram(struct xe_device *xe, unsigned int vfid, int xe_sriov_pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs); int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs); +int xe_sriov_pf_reprovision_default(struct xe_device *xe); int xe_sriov_pf_provision_set_mode(struct xe_device *xe, enum xe_sriov_provisioning_mode mode); diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_types.h index 080cf10512f4..b0253e1ae5da 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_types.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_types.h @@ -36,9 +36,6 @@ struct xe_sriov_metadata { * @XE_SRIOV_MODE_PF mode. */ struct xe_device_pf { - /** @admin_only: PF functionality focused on VFs management only. */ - bool admin_only; - /** @device_total_vfs: Maximum number of VFs supported by the device. */ u16 device_total_vfs; diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index 09b99fb2608b..6787564629c6 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -404,6 +404,8 @@ void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx) /** * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO. * @bo: the &buffer object to which batch buffer commands will be added. + * @new_mem: the (not yet committed) destination resource @bo is being moved + * into; bo->ttm.resource is still the old resource at this point. * * This function shall be called only by VF. It inserts the PTEs and copy * command instructions in the BO by calling xe_migrate_ccs_rw_copy() @@ -411,7 +413,7 @@ void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx) * * Returns: 0 if successful, negative error code on failure. */ -int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) +int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem) { struct xe_device *xe = xe_bo_device(bo); enum xe_sriov_vf_ccs_rw_ctxs ctx_id; @@ -430,7 +432,21 @@ int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) xe_assert(xe, !bb); ctx = &xe->sriov.vf.ccs.contexts[ctx_id]; - err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, ctx_id); + err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, new_mem, ctx_id); + if (err) + goto err_unwind; + } + return 0; + +err_unwind: + /* + * Clean up any contexts already attached. Can't reuse + * xe_sriov_vf_ccs_detach_bo() here as it requires both contexts + * attached before cleaning up either one. + */ + for_each_ccs_rw_ctx(ctx_id) { + if (bo->bb_ccs[ctx_id]) + xe_migrate_ccs_rw_copy_clear(bo, ctx_id); } return err; } diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h index 00e58b36c510..e1034d852104 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h @@ -11,11 +11,12 @@ #include "xe_sriov_vf_ccs_types.h" struct drm_printer; +struct ttm_resource; struct xe_device; struct xe_bo; int xe_sriov_vf_ccs_init(struct xe_device *xe); -int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo); +int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem); int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo); int xe_sriov_vf_ccs_register_context(struct xe_device *xe); void xe_sriov_vf_ccs_rebase(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_step.c b/drivers/gpu/drm/xe/xe_step.c index d0f888c31831..fb9c31613ca7 100644 --- a/drivers/gpu/drm/xe/xe_step.c +++ b/drivers/gpu/drm/xe/xe_step.c @@ -278,7 +278,7 @@ void xe_step_gmdid_get(struct xe_device *xe, case STEP_##name: \ return #name; -const char *xe_step_name(enum xe_step step) +const char *xe_step_name(enum intel_step step) { switch (step) { STEP_NAME_LIST(STEP_NAME_CASE); diff --git a/drivers/gpu/drm/xe/xe_step.h b/drivers/gpu/drm/xe/xe_step.h index 41f1c95c46e5..ea36b22cc297 100644 --- a/drivers/gpu/drm/xe/xe_step.h +++ b/drivers/gpu/drm/xe/xe_step.h @@ -18,8 +18,8 @@ void xe_step_pre_gmdid_get(struct xe_device *xe); void xe_step_gmdid_get(struct xe_device *xe, u32 graphics_gmdid_revid, u32 media_gmdid_revid); -static inline u32 xe_step_to_gmdid(enum xe_step step) { return step - STEP_A0; } +static inline u32 xe_step_to_gmdid(enum intel_step step) { return step - STEP_A0; } -const char *xe_step_name(enum xe_step step); +const char *xe_step_name(enum intel_step step); #endif diff --git a/drivers/gpu/drm/xe/xe_step_types.h b/drivers/gpu/drm/xe/xe_step_types.h index 43ca73850739..f60572b93523 100644 --- a/drivers/gpu/drm/xe/xe_step_types.h +++ b/drivers/gpu/drm/xe/xe_step_types.h @@ -8,6 +8,8 @@ #include <linux/types.h> +#include <drm/intel/step.h> + struct xe_step_info { u8 platform; u8 graphics; @@ -15,63 +17,4 @@ struct xe_step_info { u8 basedie; }; -#define STEP_ENUM_VAL(name) STEP_##name, - -/* - * Always define four minor steppings 0-3 for each stepping to match GMD ID - * spacing of values. See xe_step_gmdid_get(). - */ -#define STEP_NAME_LIST(func) \ - func(A0) \ - func(A1) \ - func(A2) \ - func(A3) \ - func(B0) \ - func(B1) \ - func(B2) \ - func(B3) \ - func(C0) \ - func(C1) \ - func(C2) \ - func(C3) \ - func(D0) \ - func(D1) \ - func(D2) \ - func(D3) \ - func(E0) \ - func(E1) \ - func(E2) \ - func(E3) \ - func(F0) \ - func(F1) \ - func(F2) \ - func(F3) \ - func(G0) \ - func(G1) \ - func(G2) \ - func(G3) \ - func(H0) \ - func(H1) \ - func(H2) \ - func(H3) \ - func(I0) \ - func(I1) \ - func(I2) \ - func(I3) \ - func(J0) \ - func(J1) \ - func(J2) \ - func(J3) - -/* - * Symbolic steppings that do not match the hardware. These are valid both as gt - * and display steppings as symbolic names. - */ -enum xe_step { - STEP_NONE = 0, - STEP_NAME_LIST(STEP_ENUM_VAL) - STEP_FUTURE, - STEP_FOREVER, -}; - #endif diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index db64cac39c94..427afd144f3a 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -396,25 +396,21 @@ bool xe_survivability_mode_is_requested(struct xe_device *xe) * Runtime survivability mode is enabled when certain errors cause the device to be * in non-recoverable state. The device is declared wedged with the appropriate * recovery method and survivability mode sysfs exposed to userspace - * - * Return: 0 if runtime survivability mode is enabled, negative error code otherwise. */ -int xe_survivability_mode_runtime_enable(struct xe_device *xe) +void xe_survivability_mode_runtime_enable(struct xe_device *xe) { struct xe_survivability *survivability = &xe->survivability; struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - int ret; if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || xe->info.platform < XE_BATTLEMAGE) { dev_err(&pdev->dev, "Runtime Survivability Mode not supported\n"); - return -EINVAL; + return; } populate_survivability_info(xe); - ret = create_survivability_sysfs(pdev); - if (ret) - dev_err(&pdev->dev, "Failed to create survivability mode sysfs\n"); + if (create_survivability_sysfs(pdev)) + dev_err(&pdev->dev, "Failed to create survivability sysfs\n"); survivability->type = XE_SURVIVABILITY_TYPE_RUNTIME; dev_err(&pdev->dev, "Runtime Survivability mode enabled\n"); @@ -422,8 +418,6 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe) xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR); xe_device_declare_wedged(xe); dev_err(&pdev->dev, "Firmware flash required, Please refer to the userspace documentation for more details!\n"); - - return 0; } /** diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.h b/drivers/gpu/drm/xe/xe_survivability_mode.h index 1cc94226aa82..cd040e4d18bb 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode.h @@ -11,7 +11,7 @@ struct xe_device; int xe_survivability_mode_boot_enable(struct xe_device *xe); -int xe_survivability_mode_runtime_enable(struct xe_device *xe); +void xe_survivability_mode_runtime_enable(struct xe_device *xe); bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe); bool xe_survivability_mode_is_requested(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 5933b2b6392b..b1e1ac26c66d 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -786,12 +786,12 @@ static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocati struct xe_bo *bo = to_xe_bo(devmem_allocation); struct ttm_resource *res = bo->ttm.resource; struct list_head *blocks = &to_xe_ttm_vram_mgr_resource(res)->blocks; + struct xe_vram_region *vr = xe_map_resource_to_region(res); + struct gpu_buddy *buddy = vram_to_buddy(vr); struct gpu_buddy_block *block; int j = 0; list_for_each_entry(block, blocks, link) { - struct xe_vram_region *vr = block->private; - struct gpu_buddy *buddy = vram_to_buddy(vr); u64 block_pfn = block_offset_to_pfn(devmem_allocation->dpagemap, gpu_buddy_block_offset(block)); int i; @@ -1055,15 +1055,12 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); struct drm_pagemap_migrate_details mdetails = { .timeslice_ms = timeslice_ms, - .source_peer_migrates = 1, }; struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap); struct dma_fence *pre_migrate_fence = NULL; struct xe_device *xe = vr->xe; struct device *dev = xe->drm.dev; - struct gpu_buddy_block *block; struct xe_validation_ctx vctx; - struct list_head *blocks; struct drm_exec exec; struct xe_bo *bo; int err = 0, idx; @@ -1100,10 +1097,6 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, &dpagemap_devmem_ops, dpagemap, end - start, pre_migrate_fence); - blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks; - list_for_each_entry(block, blocks, link) - block->private = vr; - xe_bo_get(bo); /* Ensure the device has a pm ref while there are device pages active. */ @@ -1255,10 +1248,8 @@ retry: xe_svm_range_fault_count_stats_incr(gt, range); - if (ctx.devmem_only && !range->base.pages.flags.migrate_devmem) { - err = -EACCES; - goto out; - } + if (ctx.devmem_only && !range->base.pages.flags.migrate_devmem) + return -EACCES; if (xe_svm_range_is_valid(range, tile, ctx.devmem_only, dpagemap)) { xe_svm_range_valid_fault_count_stats_incr(gt, range); diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h index b7b8eeacf196..3ca46a6f98c7 100644 --- a/drivers/gpu/drm/xe/xe_svm.h +++ b/drivers/gpu/drm/xe/xe_svm.h @@ -394,8 +394,19 @@ static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_inst #define xe_svm_assert_in_notifier(vm__) \ lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) -#define xe_svm_assert_held_read(vm__) \ +/* + * Assert the svm notifier_lock is held. Read mode by default; write mode + * when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on, because that path forces + * a userptr invalidation that ends in drm_gpusvm_unmap_pages() with + * ctx->in_notifier=true, which requires the lock held for write. + */ +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) +#define xe_svm_assert_held_read_or_inject_write(vm__) \ + lockdep_assert_held_write(&(vm__)->svm.gpusvm.notifier_lock) +#else +#define xe_svm_assert_held_read_or_inject_write(vm__) \ lockdep_assert_held_read(&(vm__)->svm.gpusvm.notifier_lock) +#endif #define xe_svm_notifier_lock(vm__) \ drm_gpusvm_notifier_lock(&(vm__)->svm.gpusvm) @@ -409,7 +420,7 @@ static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_inst #else #define xe_svm_assert_in_notifier(...) do {} while (0) -static inline void xe_svm_assert_held_read(struct xe_vm *vm) +static inline void xe_svm_assert_held_read_or_inject_write(struct xe_vm *vm) { } diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index 24d6d9af20d6..37866768d64c 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -376,7 +376,7 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, xe_assert(vm->xe, current_fence == num_fence); cf = dma_fence_array_create(num_fence, fences, dma_fence_context_alloc(1), - 1, false); + 1); if (!cf) goto err_out; diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c new file mode 100644 index 000000000000..1db20be8158b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include <linux/device.h> +#include <linux/mutex.h> + +#include <drm/drm_managed.h> + +#include "regs/xe_irq_regs.h" +#include "regs/xe_sysctrl_regs.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_soc_remapper.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_types.h" + +/** + * DOC: System Controller (sysctrl) + * + * System Controller (sysctrl) is a firmware-managed entity on Intel dGPUs + * responsible for selected low-level platform management functions. + * Communication between driver and System Controller is performed + * via a mailbox interface, enabling command and response exchange. + * + * This module provides initialization and support code for interacting + * with System Controller through the mailbox interface. + */ +static void sysctrl_fini(void *arg) +{ + struct xe_device *xe = arg; + struct xe_sysctrl *sc = &xe->sc; + + disable_work_sync(&sc->work); + xe->soc_remapper.set_sysctrl_region(xe, 0); +} + +static void xe_sysctrl_work(struct work_struct *work) +{ + struct xe_sysctrl *sc = container_of(work, struct xe_sysctrl, work); + struct xe_device *xe = sc_to_xe(sc); + + guard(xe_pm_runtime)(xe); + xe_sysctrl_event(sc); +} + +/** + * xe_sysctrl_init() - Initialize System Controller subsystem + * @xe: xe device instance + * + * Entry point for System Controller initialization, called from xe_device_probe. + * This function checks platform support and initializes the system controller. + * + * Return: 0 on success, error code on failure + */ +int xe_sysctrl_init(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_sysctrl *sc = &xe->sc; + int ret; + + if (!xe->info.has_soc_remapper_sysctrl) + return 0; + + if (!xe->info.has_sysctrl) + return 0; + + sc->mmio = devm_kzalloc(xe->drm.dev, sizeof(*sc->mmio), GFP_KERNEL); + if (!sc->mmio) + return -ENOMEM; + + xe_mmio_init(sc->mmio, tile, tile->mmio.regs, tile->mmio.regs_size); + sc->mmio->adj_offset = SYSCTRL_BASE; + sc->mmio->adj_limit = U32_MAX; + + ret = devm_mutex_init(xe->drm.dev, &sc->cmd_lock); + if (ret) + return ret; + + ret = devm_mutex_init(xe->drm.dev, &sc->event_lock); + if (ret) + return ret; + + xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); + xe_sysctrl_mailbox_init(sc); + INIT_WORK(&sc->work, xe_sysctrl_work); + + return devm_add_action_or_reset(xe->drm.dev, sysctrl_fini, xe); +} + +/** + * xe_sysctrl_irq_handler() - Handler for System Controller interrupts + * @xe: xe device instance + * @master_ctl: interrupt register + * + * Handle interrupts generated by System Controller. + */ +void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl) +{ + struct xe_sysctrl *sc = &xe->sc; + + if (!xe->info.has_sysctrl || !sc->work.func) + return; + + if (master_ctl & SYSCTRL_IRQ) + schedule_work(&sc->work); +} + +/** + * xe_sysctrl_pm_resume() - System Controller resume handler + * @xe: xe device instance + * + * Invoked during system resume (S3/S4 to S0) and runtime resume from D3cold. + * Restores SoC remapper configuration and reinitializes mailbox interface. + */ +void xe_sysctrl_pm_resume(struct xe_device *xe) +{ + struct xe_sysctrl *sc = &xe->sc; + + if (!xe->info.has_soc_remapper_sysctrl) + return; + + if (!xe->info.has_sysctrl) + return; + + xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); + + xe_sysctrl_mailbox_init(sc); +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl.h b/drivers/gpu/drm/xe/xe_sysctrl.h new file mode 100644 index 000000000000..090dffb6d55f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_H_ +#define _XE_SYSCTRL_H_ + +#include <linux/container_of.h> + +#include "xe_device_types.h" +#include "xe_sysctrl_types.h" + +static inline struct xe_device *sc_to_xe(struct xe_sysctrl *sc) +{ + return container_of(sc, struct xe_device, sc); +} + +void xe_sysctrl_event(struct xe_sysctrl *sc); +int xe_sysctrl_init(struct xe_device *xe); +void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl); +void xe_sysctrl_pm_resume(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event.c b/drivers/gpu/drm/xe/xe_sysctrl_event.c new file mode 100644 index 000000000000..b4d17329af6c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_event.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_irq.h" +#include "xe_printk.h" +#include "xe_ras.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_event_types.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" + +static void get_pending_event(struct xe_sysctrl *sc, struct xe_sysctrl_mailbox_command *command) +{ + struct xe_sysctrl_event_response *response = command->data_out; + struct xe_device *xe = sc_to_xe(sc); + u32 count = XE_SYSCTRL_EVENT_FLOOD; + size_t len; + int ret; + + do { + memset(response, 0, sizeof(*response)); + + ret = xe_sysctrl_send_command(sc, command, &len); + if (ret) { + xe_err(xe, "sysctrl: failed to get pending event %d\n", ret); + return; + } + + if (len != sizeof(*response)) { + xe_err(xe, "sysctrl: unexpected event response length %zu (expected %zu)\n", + len, sizeof(*response)); + return; + } + + if (response->event == XE_SYSCTRL_EVENT_THRESHOLD_CROSSED) + xe_ras_counter_threshold_crossed(xe, response); + else + xe_warn(xe, "sysctrl: unexpected event %#x\n", response->event); + + if (!--count) { + xe_err(xe, "sysctrl: event flooding\n"); + return; + } + + xe_dbg(xe, "sysctrl: %u events pending\n", response->count); + } while (response->count); +} + +static void event_request_prepare(struct xe_device *xe, struct xe_sysctrl_app_msg_hdr *header, + struct xe_sysctrl_event_request *request) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + header->data = REG_FIELD_PREP(APP_HDR_GROUP_ID_MASK, XE_SYSCTRL_GROUP_GFSP) | + REG_FIELD_PREP(APP_HDR_COMMAND_MASK, XE_SYSCTRL_CMD_GET_PENDING_EVENT); + + request->vector = xe_device_has_msix(xe) ? XE_IRQ_DEFAULT_MSIX : 0; + request->fn = PCI_FUNC(pdev->devfn); +} + +/** + * xe_sysctrl_event() - Handler for System Controller events + * @sc: System Controller instance + * + * Handle events generated by System Controller. + */ +void xe_sysctrl_event(struct xe_sysctrl *sc) +{ + struct xe_sysctrl_mailbox_command command = {}; + struct xe_sysctrl_event_response response = {}; + struct xe_sysctrl_event_request request = {}; + struct xe_sysctrl_app_msg_hdr header = {}; + + xe_device_assert_mem_access(sc_to_xe(sc)); + event_request_prepare(sc_to_xe(sc), &header, &request); + + command.header = header; + command.data_in = &request; + command.data_in_len = sizeof(request); + command.data_out = &response; + command.data_out_len = sizeof(response); + + guard(mutex)(&sc->event_lock); + get_pending_event(sc, &command); +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event_types.h b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h new file mode 100644 index 000000000000..c16c66b9fa7f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_EVENT_TYPES_H_ +#define _XE_SYSCTRL_EVENT_TYPES_H_ + +#include <linux/types.h> + +#define XE_SYSCTRL_EVENT_DATA_LEN 59 + +/* Modify as needed */ +#define XE_SYSCTRL_EVENT_FLOOD 16 + +/** + * enum xe_sysctrl_event - Events reported by System Controller + * + * @XE_SYSCTRL_EVENT_THRESHOLD_CROSSED: Error counter threshold crossed + */ +enum xe_sysctrl_event { + XE_SYSCTRL_EVENT_THRESHOLD_CROSSED = 0x01, +}; + +/** + * struct xe_sysctrl_event_request - Request structure for pending event + */ +struct xe_sysctrl_event_request { + /** @vector: MSI-X vector that was triggered */ + u32 vector; + /** @fn: Function index (0-7) of PCIe device */ + u32 fn:8; + /** @reserved: Reserved for future use */ + u32 reserved:24; + /** @reserved1: Reserved for future use */ + u32 reserved1[2]; +} __packed; + +/** + * struct xe_sysctrl_event_response - Response structure for pending event + */ +struct xe_sysctrl_event_response { + /** @count: Pending event count after this response */ + u32 count; + /** @event: Pending event type */ + u32 event; + /** @timestamp: Timestamp of most recent event */ + u64 timestamp; + /** @extended: Event has extended payload */ + u32 extended:1; + /** @reserved: Reserved for future use */ + u32 reserved:31; + /** @data: Generic event data */ + u32 data[XE_SYSCTRL_EVENT_DATA_LEN]; +} __packed; + +#endif /* _XE_SYSCTRL_EVENT_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c new file mode 100644 index 000000000000..3caa9f15875f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include <linux/bitfield.h> +#include <linux/cleanup.h> +#include <linux/minmax.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "regs/xe_sysctrl_regs.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_printk.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" + +struct xe_sysctrl_mailbox_msg_hdr { + __le32 data; +} __packed; + +#define XE_SYSCTRL_HDR_GROUP_ID(hdr) \ + FIELD_GET(SYSCTRL_HDR_GROUP_ID_MASK, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_COMMAND(hdr) \ + FIELD_GET(SYSCTRL_HDR_COMMAND_MASK, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_IS_RESPONSE(hdr) \ + FIELD_GET(SYSCTRL_HDR_IS_RESPONSE, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_RESULT(hdr) \ + FIELD_GET(SYSCTRL_HDR_RESULT_MASK, le32_to_cpu((hdr)->data)) + +static bool sysctrl_wait_bit_clear(struct xe_sysctrl *sc, u32 bit_mask, + unsigned int timeout_ms) +{ + int ret; + + ret = xe_mmio_wait32_not(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask, + timeout_ms * 1000, NULL, false); + + return ret == 0; +} + +static bool sysctrl_wait_bit_set(struct xe_sysctrl *sc, u32 bit_mask, + unsigned int timeout_ms) +{ + int ret; + + ret = xe_mmio_wait32(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask, + timeout_ms * 1000, NULL, false); + + return ret == 0; +} + +static int sysctrl_write_frame(struct xe_sysctrl *sc, const void *frame, + size_t len) +{ + static const struct xe_reg regs[] = { + SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3 + }; + struct xe_device *xe = sc_to_xe(sc); + u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0}; + u32 dw = DIV_ROUND_UP(len, sizeof(u32)); + u32 i; + + xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE); + + memcpy(val, frame, len); + + for (i = 0; i < dw; i++) + xe_mmio_write32(sc->mmio, regs[i], val[i]); + + return 0; +} + +static int sysctrl_read_frame(struct xe_sysctrl *sc, void *frame, + size_t len) +{ + static const struct xe_reg regs[] = { + SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3 + }; + struct xe_device *xe = sc_to_xe(sc); + u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0}; + u32 dw = DIV_ROUND_UP(len, sizeof(u32)); + u32 i; + + xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE); + + for (i = 0; i < dw; i++) + val[i] = xe_mmio_read32(sc->mmio, regs[i]); + + memcpy(frame, val, len); + + return 0; +} + +static void sysctrl_clear_response(struct xe_sysctrl *sc) +{ + xe_mmio_rmw32(sc->mmio, SYSCTRL_MB_CTRL, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, 0); +} + +static int sysctrl_prepare_command(struct xe_device *xe, + u8 group_id, u8 command, + const void *data_in, size_t data_in_len, + u8 **mbox_cmd, size_t *cmd_size) +{ + struct xe_sysctrl_mailbox_msg_hdr *hdr; + size_t size; + u8 *buffer; + + xe_assert(xe, command <= SYSCTRL_HDR_COMMAND_MAX); + + if (data_in_len > XE_SYSCTRL_MB_MAX_MESSAGE_SIZE - sizeof(*hdr)) { + xe_err(xe, "sysctrl: Input data too large: %zu bytes\n", data_in_len); + return -EINVAL; + } + + size = sizeof(*hdr) + data_in_len; + + buffer = kmalloc(size, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer; + hdr->data = cpu_to_le32(FIELD_PREP(SYSCTRL_HDR_GROUP_ID_MASK, group_id) | + FIELD_PREP(SYSCTRL_HDR_COMMAND_MASK, command)); + + if (data_in && data_in_len) + memcpy(buffer + sizeof(*hdr), data_in, data_in_len); + + *mbox_cmd = buffer; + *cmd_size = size; + + return 0; +} + +static int sysctrl_send_frames(struct xe_sysctrl *sc, + const u8 *mbox_cmd, + size_t cmd_size, unsigned int timeout_ms) +{ + struct xe_device *xe = sc_to_xe(sc); + u32 ctrl_reg, total_frames, frame; + size_t bytes_sent, frame_size; + + total_frames = DIV_ROUND_UP(cmd_size, XE_SYSCTRL_MB_FRAME_SIZE); + + if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) { + xe_err(xe, "sysctrl: Mailbox busy\n"); + return -EBUSY; + } + + sc->phase_bit ^= 1; + bytes_sent = 0; + + for (frame = 0; frame < total_frames; frame++) { + frame_size = min_t(size_t, cmd_size - bytes_sent, XE_SYSCTRL_MB_FRAME_SIZE); + + if (sysctrl_write_frame(sc, mbox_cmd + bytes_sent, frame_size)) { + xe_err(xe, "sysctrl: Failed to write frame %u\n", frame); + sc->phase_bit = 0; + return -EIO; + } + + ctrl_reg = SYSCTRL_MB_CTRL_RUN_BUSY | + REG_FIELD_PREP(SYSCTRL_FRAME_CURRENT_MASK, frame) | + REG_FIELD_PREP(SYSCTRL_FRAME_TOTAL_MASK, total_frames - 1) | + SYSCTRL_MB_CTRL_CMD | + (sc->phase_bit ? SYSCTRL_FRAME_PHASE : 0); + + xe_mmio_write32(sc->mmio, SYSCTRL_MB_CTRL, ctrl_reg); + + if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) { + xe_err(xe, "sysctrl: Frame %u acknowledgment timeout\n", frame); + sc->phase_bit = 0; + return -ETIMEDOUT; + } + + bytes_sent += frame_size; + } + + return 0; +} + +static int sysctrl_process_frame(struct xe_sysctrl *sc, void *out, + size_t frame_size, unsigned int timeout_ms, + bool *done) +{ + u32 curr_frame, total_frames, ctrl_reg; + struct xe_device *xe = sc_to_xe(sc); + int ret; + + if (!sysctrl_wait_bit_set(sc, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, timeout_ms)) { + xe_err(xe, "sysctrl: Response frame timeout\n"); + return -ETIMEDOUT; + } + + ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); + total_frames = FIELD_GET(SYSCTRL_FRAME_TOTAL_MASK, ctrl_reg); + curr_frame = FIELD_GET(SYSCTRL_FRAME_CURRENT_MASK, ctrl_reg); + + ret = sysctrl_read_frame(sc, out, frame_size); + if (ret) + return ret; + + sysctrl_clear_response(sc); + + if (curr_frame == total_frames) + *done = true; + + return 0; +} + +static int sysctrl_receive_frames(struct xe_sysctrl *sc, + const struct xe_sysctrl_mailbox_msg_hdr *req, + void *data_out, size_t data_out_len, + size_t *rdata_len, unsigned int timeout_ms) +{ + struct xe_sysctrl_mailbox_msg_hdr *hdr; + struct xe_device *xe = sc_to_xe(sc); + size_t remain = sizeof(*hdr) + data_out_len; + u8 *buffer __free(kfree) = kzalloc(remain, GFP_KERNEL); + size_t frame_size; + bool done = false; + int ret = 0; + u8 *out; + + if (!buffer) + return -ENOMEM; + + out = buffer; + while (!done && remain) { + frame_size = min_t(size_t, remain, XE_SYSCTRL_MB_FRAME_SIZE); + + ret = sysctrl_process_frame(sc, out, frame_size, timeout_ms, + &done); + if (ret) + return ret; + + remain -= frame_size; + out += frame_size; + } + + hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer; + + if (!XE_SYSCTRL_HDR_IS_RESPONSE(hdr) || + XE_SYSCTRL_HDR_GROUP_ID(hdr) != XE_SYSCTRL_HDR_GROUP_ID(req) || + XE_SYSCTRL_HDR_COMMAND(hdr) != XE_SYSCTRL_HDR_COMMAND(req)) { + xe_err(xe, "sysctrl: Response header mismatch\n"); + return -EPROTO; + } + + if (XE_SYSCTRL_HDR_RESULT(hdr) != 0) { + xe_err(xe, "sysctrl: Firmware error: 0x%02lx\n", + XE_SYSCTRL_HDR_RESULT(hdr)); + return -EIO; + } + + memcpy(data_out, hdr + 1, data_out_len); + *rdata_len = out - buffer - sizeof(*hdr); + + return 0; +} + +static int sysctrl_send_command(struct xe_sysctrl *sc, + const u8 *mbox_cmd, size_t cmd_size, + void *data_out, size_t data_out_len, + size_t *rdata_len, unsigned int timeout_ms) +{ + const struct xe_sysctrl_mailbox_msg_hdr *hdr; + size_t received; + int ret; + + ret = sysctrl_send_frames(sc, mbox_cmd, cmd_size, timeout_ms); + if (ret) + return ret; + + if (!data_out || !rdata_len) + return 0; + + hdr = (const struct xe_sysctrl_mailbox_msg_hdr *)mbox_cmd; + + ret = sysctrl_receive_frames(sc, hdr, data_out, data_out_len, + &received, timeout_ms); + if (ret) + return ret; + + *rdata_len = received; + + return 0; +} + +/** + * xe_sysctrl_mailbox_init - Initialize System Controller mailbox interface + * @sc: System controller structure + * + * Initialize system controller mailbox interface for communication. + */ +void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc) +{ + u32 ctrl_reg; + + ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); + sc->phase_bit = (ctrl_reg & SYSCTRL_FRAME_PHASE) ? 1 : 0; +} + +/** + * xe_sysctrl_send_command() - Send mailbox command to System Controller + * @sc: System Controller instance + * @cmd: Command descriptor containing request header and payload buffers + * @rdata_len: Pointer to store actual response data length + * + * Sends a mailbox command to System Controller firmware using + * System Controller mailbox and waits for a response. + * + * Request payload is provided via @cmd->data_in and @cmd->data_in_len. + * If a response is expected, @cmd->data_out must point to a buffer of + * size @cmd->data_out_len supplied by caller. + * + * On success, @rdata_len is updated with number of valid response bytes + * returned by firmware, bounded by @cmd->data_out_len. + * + * Return: 0 on success, or negative errno on failure. + */ +int xe_sysctrl_send_command(struct xe_sysctrl *sc, + struct xe_sysctrl_mailbox_command *cmd, + size_t *rdata_len) +{ + struct xe_device *xe = sc_to_xe(sc); + u8 group_id, command_code; + u8 *mbox_cmd = NULL; + size_t cmd_size = 0; + int ret; + + guard(xe_pm_runtime_noresume)(xe); + + if (!xe->info.has_sysctrl) + return -ENODEV; + + xe_assert(xe, cmd->data_in || cmd->data_out); + xe_assert(xe, !cmd->data_in || cmd->data_in_len); + xe_assert(xe, !cmd->data_out || cmd->data_out_len); + + group_id = XE_SYSCTRL_APP_HDR_GROUP_ID(&cmd->header); + command_code = XE_SYSCTRL_APP_HDR_COMMAND(&cmd->header); + + might_sleep(); + + ret = sysctrl_prepare_command(xe, group_id, command_code, + cmd->data_in, cmd->data_in_len, + &mbox_cmd, &cmd_size); + if (ret) { + xe_err(xe, "sysctrl: Failed to prepare command: %pe\n", ERR_PTR(ret)); + return ret; + } + + guard(mutex)(&sc->cmd_lock); + + ret = sysctrl_send_command(sc, mbox_cmd, cmd_size, + cmd->data_out, cmd->data_out_len, rdata_len, + XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS); + if (ret) + xe_err(xe, "sysctrl: Mailbox command failed: %pe\n", ERR_PTR(ret)); + + kfree(mbox_cmd); + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h new file mode 100644 index 000000000000..f67e9234de48 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_MAILBOX_H_ +#define _XE_SYSCTRL_MAILBOX_H_ + +#include <linux/bitfield.h> +#include <linux/types.h> + +#include "abi/xe_sysctrl_abi.h" + +struct xe_sysctrl; +struct xe_sysctrl_mailbox_command; + +#define XE_SYSCTRL_APP_HDR_GROUP_ID(hdr) \ + FIELD_GET(APP_HDR_GROUP_ID_MASK, (hdr)->data) + +#define XE_SYSCTRL_APP_HDR_COMMAND(hdr) \ + FIELD_GET(APP_HDR_COMMAND_MASK, (hdr)->data) + +#define XE_SYSCTRL_APP_HDR_VERSION(hdr) \ + FIELD_GET(APP_HDR_VERSION_MASK, (hdr)->data) + +void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc); +int xe_sysctrl_send_command(struct xe_sysctrl *sc, + struct xe_sysctrl_mailbox_command *cmd, + size_t *rdata_len); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h new file mode 100644 index 000000000000..84d7c647e743 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_MAILBOX_TYPES_H_ +#define _XE_SYSCTRL_MAILBOX_TYPES_H_ + +#include <linux/types.h> + +#include "abi/xe_sysctrl_abi.h" + +/** + * enum xe_sysctrl_group - System Controller command groups + * + * @XE_SYSCTRL_GROUP_GFSP: GFSP group + */ +enum xe_sysctrl_group { + XE_SYSCTRL_GROUP_GFSP = 0x01, +}; + +/** + * enum xe_sysctrl_gfsp_cmd - Commands supported by GFSP group + * + * @XE_SYSCTRL_CMD_GET_PENDING_EVENT: Retrieve pending event + */ +enum xe_sysctrl_gfsp_cmd { + XE_SYSCTRL_CMD_GET_PENDING_EVENT = 0x07, +}; + +/** + * struct xe_sysctrl_mailbox_command - System Controller mailbox command + */ +struct xe_sysctrl_mailbox_command { + /** @header: Application message header containing command information */ + struct xe_sysctrl_app_msg_hdr header; + + /** @data_in: Pointer to input payload data (can be NULL if no input data) */ + void *data_in; + + /** @data_in_len: Size of input payload in bytes (0 if no input data) */ + size_t data_in_len; + + /** @data_out: Pointer to output buffer for response data (can be NULL if no response) */ + void *data_out; + + /** @data_out_len: Size of output buffer in bytes (0 if no response expected) */ + size_t data_out_len; +}; + +#define XE_SYSCTRL_MB_FRAME_SIZE 16 +#define XE_SYSCTRL_MB_MAX_FRAMES 64 +#define XE_SYSCTRL_MB_MAX_MESSAGE_SIZE \ + (XE_SYSCTRL_MB_FRAME_SIZE * XE_SYSCTRL_MB_MAX_FRAMES) + +#define XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS 500 + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_types.h b/drivers/gpu/drm/xe/xe_sysctrl_types.h new file mode 100644 index 000000000000..66ba24f43017 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_types.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_TYPES_H_ +#define _XE_SYSCTRL_TYPES_H_ + +#include <linux/mutex.h> +#include <linux/types.h> +#include <linux/workqueue_types.h> + +struct xe_mmio; + +/** + * struct xe_sysctrl - System Controller driver context + * + * This structure maintains the runtime state for System Controller + * communication. All fields are initialized during xe_sysctrl_init() + * and protected appropriately for concurrent access. + */ +struct xe_sysctrl { + /** @mmio: MMIO region for system control registers */ + struct xe_mmio *mmio; + + /** @cmd_lock: Mutex protecting mailbox command operations */ + struct mutex cmd_lock; + + /** @phase_bit: Message boundary phase toggle bit (0 or 1) */ + bool phase_bit; + + /** @work: Pending events worker */ + struct work_struct work; + + /** @event_lock: Mutex protecting pending events */ + struct mutex event_lock; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index 10dcd4abb00f..bbd21d393062 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -529,7 +529,7 @@ int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid, struct xe_tile *tile; u32 fence_id = 0; u8 id; - int err; + int err = 0; batch->num_fences = 0; if (!tile_mask) diff --git a/drivers/gpu/drm/xe/xe_trace_lrc.h b/drivers/gpu/drm/xe/xe_trace_lrc.h index d525cbee1e34..5c4cfa0c1fe9 100644 --- a/drivers/gpu/drm/xe/xe_trace_lrc.h +++ b/drivers/gpu/drm/xe/xe_trace_lrc.h @@ -12,6 +12,7 @@ #include <linux/tracepoint.h> #include <linux/types.h> +#include "xe_exec_queue_types.h" #include "xe_gt_types.h" #include "xe_lrc.h" #include "xe_lrc_types.h" @@ -42,6 +43,32 @@ TRACE_EVENT(xe_lrc_update_timestamp, __get_str(device_id)) ); +TRACE_EVENT(xe_lrc_update_queue_timestamp, + TP_PROTO(struct xe_lrc *lrc, uint64_t old), + TP_ARGS(lrc, old), + TP_STRUCT__entry( + __field(struct xe_lrc *, lrc) + __field(struct xe_lrc *, primary_lrc) + __field(u64, old) + __field(u64, new) + __string(name, lrc->fence_ctx.name) + __string(device_id, __dev_name_lrc(lrc)) + ), + + TP_fast_assign( + __entry->lrc = lrc; + __entry->primary_lrc = lrc->multi_queue.primary_lrc; + __entry->old = old; + __entry->new = lrc->queue_timestamp; + __assign_str(name); + __assign_str(device_id); + ), + TP_printk("lrc=%p primary_lrc=%p lrc->name=%s old=%llu new=%llu device_id:%s", + __entry->lrc, __entry->primary_lrc, __get_str(name), + __entry->old, __entry->new, + __get_str(device_id)) +); + #endif /* This part must be outside protection */ diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c index 27c9d72222cf..5e9070739e65 100644 --- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c @@ -19,30 +19,11 @@ #include "xe_device.h" #include "xe_gt_printk.h" #include "xe_mmio.h" -#include "xe_res_cursor.h" #include "xe_sriov.h" #include "xe_ttm_stolen_mgr.h" -#include "xe_ttm_vram_mgr.h" #include "xe_vram.h" #include "xe_wa.h" -struct xe_ttm_stolen_mgr { - struct xe_ttm_vram_mgr base; - - /* PCI base offset */ - resource_size_t io_base; - /* GPU base offset */ - resource_size_t stolen_base; - - void __iomem *mapping; -}; - -static inline struct xe_ttm_stolen_mgr * -to_stolen_mgr(struct ttm_resource_manager *man) -{ - return container_of(man, struct xe_ttm_stolen_mgr, base.manager); -} - /** * xe_ttm_stolen_cpu_access_needs_ggtt() - If we can't directly CPU access * stolen, can we then fallback to mapping through the GGTT. @@ -210,12 +191,19 @@ static u64 detect_stolen(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr) #endif } +static void xe_ttm_stolen_mgr_fini(struct drm_device *dev, void *arg) +{ + struct xe_device *xe = to_xe_device(dev); + + ttm_range_man_fini_nocheck(&xe->ttm, XE_PL_STOLEN); +} + int xe_ttm_stolen_mgr_init(struct xe_device *xe) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); struct xe_ttm_stolen_mgr *mgr; u64 stolen_size, io_size; - int err; + int ret; mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL); if (!mgr) @@ -244,12 +232,12 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe) if (mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe)) io_size = stolen_size; - err = __xe_ttm_vram_mgr_init(xe, &mgr->base, XE_PL_STOLEN, stolen_size, - io_size, PAGE_SIZE); - if (err) { - drm_dbg_kms(&xe->drm, "Stolen mgr init failed: %i\n", err); - return err; - } + ret = ttm_range_man_init_nocheck(&xe->ttm, XE_PL_STOLEN, false, + stolen_size >> PAGE_SHIFT); + if (ret) + return ret; + + xe->mem.stolen_mgr = mgr; drm_dbg_kms(&xe->drm, "Initialized stolen memory support with %llu bytes\n", stolen_size); @@ -257,36 +245,32 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe) if (io_size) mgr->mapping = devm_ioremap_wc(&pdev->dev, mgr->io_base, io_size); - return 0; + return drmm_add_action_or_reset(&xe->drm, xe_ttm_stolen_mgr_fini, mgr); } u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset) { struct xe_device *xe = xe_bo_device(bo); - struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); - struct xe_ttm_stolen_mgr *mgr = to_stolen_mgr(ttm_mgr); - struct xe_res_cursor cur; + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; XE_WARN_ON(!mgr->io_base); if (xe_ttm_stolen_cpu_access_needs_ggtt(xe)) return mgr->io_base + xe_bo_ggtt_addr(bo) + offset; - xe_res_first(bo->ttm.resource, offset, 4096, &cur); - return mgr->io_base + cur.start; + /* Range allocator: res->start is in pages. */ + return mgr->io_base + (bo->ttm.resource->start << PAGE_SHIFT) + offset; } static int __xe_ttm_stolen_io_mem_reserve_bar2(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr, struct ttm_resource *mem) { - struct xe_res_cursor cur; - if (!mgr->io_base) return -EIO; - xe_res_first(mem, 0, 4096, &cur); - mem->bus.offset = cur.start; + /* Range allocator always produces contiguous allocations. */ + mem->bus.offset = mem->start << PAGE_SHIFT; drm_WARN_ON(&xe->drm, !(mem->placement & TTM_PL_FLAG_CONTIGUOUS)); @@ -329,8 +313,7 @@ static int __xe_ttm_stolen_io_mem_reserve_stolen(struct xe_device *xe, int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem) { - struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); - struct xe_ttm_stolen_mgr *mgr = ttm_mgr ? to_stolen_mgr(ttm_mgr) : NULL; + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; if (!mgr || !mgr->io_base) return -EIO; @@ -343,8 +326,5 @@ int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem) u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe) { - struct xe_ttm_stolen_mgr *mgr = - to_stolen_mgr(ttm_manager_type(&xe->ttm, XE_PL_STOLEN)); - - return mgr->stolen_base; + return xe->mem.stolen_mgr->stolen_base; } diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h index 8e877d1e839b..0675106d535b 100644 --- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h @@ -12,6 +12,18 @@ struct ttm_resource; struct xe_bo; struct xe_device; +/** + * struct xe_ttm_stolen_mgr - Xe TTM stolen memory manager + */ +struct xe_ttm_stolen_mgr { + /** @io_base: PCI base offset for CPU I/O access */ + resource_size_t io_base; + /** @stolen_base: GPU base offset */ + resource_size_t stolen_base; + /** @mapping: I/O memory mapping for CPU access */ + void __iomem *mapping; +}; + int xe_ttm_stolen_mgr_init(struct xe_device *xe); int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem); bool xe_ttm_stolen_cpu_access_needs_ggtt(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 5fd0d5506a7e..b518f7dec680 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -292,8 +292,6 @@ static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg) ttm_resource_manager_cleanup(&mgr->manager); ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL); - - mutex_destroy(&mgr->lock); } int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, @@ -301,18 +299,19 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, u64 default_page_size) { struct ttm_resource_manager *man = &mgr->manager; + const char *name; int err; - if (mem_type != XE_PL_STOLEN) { - const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; - man->cg = drmm_cgroup_register_region(&xe->drm, name, size); - if (IS_ERR(man->cg)) - return PTR_ERR(man->cg); - } + name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; + man->cg = drmm_cgroup_register_region(&xe->drm, name, size); + if (IS_ERR(man->cg)) + return PTR_ERR(man->cg); man->func = &xe_ttm_vram_mgr_func; mgr->mem_type = mem_type; - mutex_init(&mgr->lock); + err = drmm_mutex_init(&xe->drm, &mgr->lock); + if (err) + return err; mgr->default_page_size = default_page_size; mgr->visible_size = io_size; mgr->visible_avail = io_size; @@ -322,6 +321,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, if (err) return err; + gpu_buddy_driver_set_lock(&mgr->mm, &mgr->lock); ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager); ttm_resource_manager_set_used(&mgr->manager, true); diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c index fcb6698abc6e..bf3fad9cdbef 100644 --- a/drivers/gpu/drm/xe/xe_tuning.c +++ b/drivers/gpu/drm/xe/xe_tuning.c @@ -20,7 +20,7 @@ #undef XE_REG_MCR #define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1) -static const struct xe_rtp_entry_sr gt_tunings[] = { +static const struct xe_rtp_table_sr gt_tunings = XE_RTP_TABLE_SR( { XE_RTP_NAME("Tuning: Blend Fill Caching Optimization Disable"), XE_RTP_RULES(PLATFORM(DG2)), XE_RTP_ACTIONS(SET(XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS)) @@ -43,7 +43,7 @@ static const struct xe_rtp_entry_sr gt_tunings[] = { REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f))) }, { XE_RTP_NAME("Tuning: Compression Overfetch"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3499), FUNC(xe_rtp_match_has_flat_ccs)), XE_RTP_ACTIONS(CLR(CCCHKNREG1, ENCOMPPERFFIX), SET(CCCHKNREG1, L3CMPCTRL)) @@ -100,9 +100,9 @@ static const struct xe_rtp_entry_sr gt_tunings[] = { XE_RTP_ACTIONS(FIELD_SET(GAMSTLB_CTRL, BANK_HASH_MODE, BANK_HASH_4KB_MODE)) }, -}; +); -static const struct xe_rtp_entry_sr engine_tunings[] = { +static const struct xe_rtp_table_sr engine_tunings = XE_RTP_TABLE_SR( { XE_RTP_NAME("Tuning: L3 Hashing Mask"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), FUNC(xe_rtp_match_first_render_or_compute)), @@ -124,9 +124,14 @@ static const struct xe_rtp_entry_sr engine_tunings[] = { GHWSP_CSB_REPORT_DIS, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, -}; + { XE_RTP_NAME("Tuning: TileY 2x2 Walk"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN2, TILEY_LOCALID)) + }, +); -static const struct xe_rtp_entry_sr lrc_tunings[] = { +static const struct xe_rtp_table_sr lrc_tunings = XE_RTP_TABLE_SR( { XE_RTP_NAME("Tuning: Windower HW Filtering"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3599), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, HW_FILTERING)) @@ -166,7 +171,7 @@ static const struct xe_rtp_entry_sr lrc_tunings[] = { XE_RTP_ACTIONS(FIELD_SET(FF_MODE, VS_HIT_MAX_VALUE_MASK, REG_FIELD_PREP(VS_HIT_MAX_VALUE_MASK, 0x3f))) }, -}; +); /** * xe_tuning_init - initialize gt with tunings bookkeeping @@ -180,9 +185,9 @@ int xe_tuning_init(struct xe_gt *gt) size_t n_lrc, n_engine, n_gt, total; unsigned long *p; - n_gt = BITS_TO_LONGS(ARRAY_SIZE(gt_tunings)); - n_engine = BITS_TO_LONGS(ARRAY_SIZE(engine_tunings)); - n_lrc = BITS_TO_LONGS(ARRAY_SIZE(lrc_tunings)); + n_gt = BITS_TO_LONGS(gt_tunings.n_entries); + n_engine = BITS_TO_LONGS(engine_tunings.n_entries); + n_lrc = BITS_TO_LONGS(lrc_tunings.n_entries); total = n_gt + n_engine + n_lrc; p = drmm_kzalloc(&xe->drm, sizeof(*p) * total, GFP_KERNEL); @@ -205,9 +210,8 @@ void xe_tuning_process_gt(struct xe_gt *gt) xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->tuning_active.gt, - ARRAY_SIZE(gt_tunings)); - xe_rtp_process_to_sr(&ctx, gt_tunings, ARRAY_SIZE(gt_tunings), - >->reg_sr, false); + gt_tunings.n_entries); + xe_rtp_process_to_sr(&ctx, >_tunings, >->reg_sr, false); } EXPORT_SYMBOL_IF_KUNIT(xe_tuning_process_gt); @@ -217,9 +221,8 @@ void xe_tuning_process_engine(struct xe_hw_engine *hwe) xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->tuning_active.engine, - ARRAY_SIZE(engine_tunings)); - xe_rtp_process_to_sr(&ctx, engine_tunings, ARRAY_SIZE(engine_tunings), - &hwe->reg_sr, false); + engine_tunings.n_entries); + xe_rtp_process_to_sr(&ctx, &engine_tunings, &hwe->reg_sr, false); } EXPORT_SYMBOL_IF_KUNIT(xe_tuning_process_engine); @@ -237,9 +240,8 @@ void xe_tuning_process_lrc(struct xe_hw_engine *hwe) xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->tuning_active.lrc, - ARRAY_SIZE(lrc_tunings)); - xe_rtp_process_to_sr(&ctx, lrc_tunings, ARRAY_SIZE(lrc_tunings), - &hwe->reg_lrc, true); + lrc_tunings.n_entries); + xe_rtp_process_to_sr(&ctx, &lrc_tunings, &hwe->reg_lrc, true); } /** @@ -254,18 +256,18 @@ int xe_tuning_dump(struct xe_gt *gt, struct drm_printer *p) size_t idx; drm_printf(p, "GT Tunings\n"); - for_each_set_bit(idx, gt->tuning_active.gt, ARRAY_SIZE(gt_tunings)) - drm_printf_indent(p, 1, "%s\n", gt_tunings[idx].name); + for_each_set_bit(idx, gt->tuning_active.gt, gt_tunings.n_entries) + drm_printf_indent(p, 1, "%s\n", gt_tunings.entries[idx].name); drm_puts(p, "\n"); drm_printf(p, "Engine Tunings\n"); - for_each_set_bit(idx, gt->tuning_active.engine, ARRAY_SIZE(engine_tunings)) - drm_printf_indent(p, 1, "%s\n", engine_tunings[idx].name); + for_each_set_bit(idx, gt->tuning_active.engine, engine_tunings.n_entries) + drm_printf_indent(p, 1, "%s\n", engine_tunings.entries[idx].name); drm_puts(p, "\n"); drm_printf(p, "LRC Tunings\n"); - for_each_set_bit(idx, gt->tuning_active.lrc, ARRAY_SIZE(lrc_tunings)) - drm_printf_indent(p, 1, "%s\n", lrc_tunings[idx].name); + for_each_set_bit(idx, gt->tuning_active.lrc, lrc_tunings.n_entries) + drm_printf_indent(p, 1, "%s\n", lrc_tunings.entries[idx].name); return 0; } diff --git a/drivers/gpu/drm/xe/xe_tuning.h b/drivers/gpu/drm/xe/xe_tuning.h index c1cc5927fda7..d18e187debf6 100644 --- a/drivers/gpu/drm/xe/xe_tuning.h +++ b/drivers/gpu/drm/xe/xe_tuning.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_TUNING_ -#define _XE_TUNING_ +#ifndef _XE_TUNING_H_ +#define _XE_TUNING_H_ struct drm_printer; struct xe_gt; diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c index 18ebefd444fe..3f08a3b54062 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.c +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -213,6 +213,17 @@ static struct xe_device *uc_fw_to_xe(struct xe_uc_fw *uc_fw) return gt_to_xe(uc_fw_to_gt(uc_fw)); } +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) +void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status) +{ + xe_gt_dbg(uc_fw_to_gt(uc_fw), "%s %s->%s\n", + xe_uc_fw_type_repr(uc_fw->type), + xe_uc_fw_status_repr(uc_fw->status), + xe_uc_fw_status_repr(status)); + uc_fw->__status = status; +} +#endif + static void uc_fw_auto_select(struct xe_device *xe, struct xe_uc_fw *uc_fw) { diff --git a/drivers/gpu/drm/xe/xe_uc_fw.h b/drivers/gpu/drm/xe/xe_uc_fw.h index 6195e353f269..f2d3a3e7208b 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.h +++ b/drivers/gpu/drm/xe/xe_uc_fw.h @@ -8,8 +8,8 @@ #include <linux/errno.h> +#include "abi/uc_fw_abi.h" #include "xe_macros.h" -#include "xe_uc_fw_abi.h" #include "xe_uc_fw_types.h" struct drm_printer; @@ -25,11 +25,15 @@ static inline u32 xe_uc_fw_rsa_offset(struct xe_uc_fw *uc_fw) return sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->css_offset; } +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) +void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status); +#else static inline void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status) { uc_fw->__status = status; } +#endif static inline const char *xe_uc_fw_status_repr(enum xe_uc_fw_status status) diff --git a/drivers/gpu/drm/xe/xe_userptr.c b/drivers/gpu/drm/xe/xe_userptr.c index 6761005c0b90..6f71bc66b14e 100644 --- a/drivers/gpu/drm/xe/xe_userptr.c +++ b/drivers/gpu/drm/xe/xe_userptr.c @@ -269,7 +269,7 @@ static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { */ void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma) { - static struct mmu_interval_notifier_finish *finish; + struct mmu_interval_notifier_finish *finish; struct xe_vm *vm = xe_vma_vm(&uvma->vma); /* Protect against concurrent userptr pinning */ diff --git a/drivers/gpu/drm/xe/xe_validation.c b/drivers/gpu/drm/xe/xe_validation.c index a611438eaafe..8dff4d0ec895 100644 --- a/drivers/gpu/drm/xe/xe_validation.c +++ b/drivers/gpu/drm/xe/xe_validation.c @@ -156,7 +156,7 @@ int xe_validation_ctx_init(struct xe_validation_ctx *ctx, struct xe_validation_d #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH /* - * This abuses both drm_exec and ww_mutex internals and should be + * This abuses ww_mutex internals and should be * replaced by checking for -EDEADLK when we can make TTM * stop converting -EDEADLK to -ENOMEM. * An alternative is to not have exhaustive eviction with @@ -164,7 +164,7 @@ int xe_validation_ctx_init(struct xe_validation_ctx *ctx, struct xe_validation_d */ static bool xe_validation_contention_injected(struct drm_exec *exec) { - return !!exec->ticket.contending_lock; + return !!drm_exec_ticket(exec)->contending_lock; } #else diff --git a/drivers/gpu/drm/xe/xe_validation.h b/drivers/gpu/drm/xe/xe_validation.h index a30e732c4d51..4cd955ce6cd2 100644 --- a/drivers/gpu/drm/xe/xe_validation.h +++ b/drivers/gpu/drm/xe/xe_validation.h @@ -146,7 +146,7 @@ bool xe_validation_should_retry(struct xe_validation_ctx *ctx, int *ret); #define xe_validation_retry_on_oom(_ctx, _ret) \ do { \ if (xe_validation_should_retry(_ctx, _ret)) \ - goto *__drm_exec_retry_ptr; \ + drm_exec_retry((_ctx)->exec); \ } while (0) /** diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index ab6cc1f0a789..32ded13491ca 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -373,7 +373,6 @@ int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, unsigned int num_fences) { struct drm_gem_object *obj; - unsigned long index; int ret; do { @@ -386,7 +385,7 @@ int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, return ret; } while (!list_empty(&vm->gpuvm.evict.list)); - drm_exec_for_each_locked_object(exec, index, obj) { + drm_exec_for_each_locked_object(exec, obj) { ret = dma_resv_reserve_fences(obj->resv, num_fences); if (ret) return ret; @@ -634,9 +633,9 @@ void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf) e->address_precision = SZ_4K; e->access_type = pf->consumer.access_type; e->fault_type = FIELD_GET(XE_PAGEFAULT_TYPE_MASK, - pf->consumer.fault_type_level), + pf->consumer.fault_type_level); e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK, - pf->consumer.fault_type_level), + pf->consumer.fault_type_level); list_add_tail(&e->list, &vm->faults.list); vm->faults.len++; @@ -1425,9 +1424,9 @@ static u16 pde_pat_index(struct xe_bo *bo) * something which is always safe). */ if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached) - pat_index = xe->pat.idx[XE_CACHE_WB]; + pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); else - pat_index = xe->pat.idx[XE_CACHE_NONE]; + pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE); xe_assert(xe, pat_index <= 3); @@ -3256,11 +3255,26 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, .request_decompress = false, .check_purged = true, }); - if (!err && !xe_vma_has_no_bo(vma)) - err = xe_bo_migrate(xe_vma_bo(vma), - region_to_mem_type[region], - NULL, - exec); + if (!err && !xe_vma_has_no_bo(vma)) { + struct xe_bo *bo = xe_vma_bo(vma); + u32 mem_type; + + if (region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { + unsigned int i; + + mem_type = XE_PL_TT; + for (i = 0; i < bo->placement.num_placement; i++) { + if (mem_type_is_vram(bo->placements[i].mem_type)) { + mem_type = bo->placements[i].mem_type; + break; + } + } + } else { + mem_type = region_to_mem_type[region]; + } + + err = xe_bo_migrate(bo, mem_type, NULL, exec); + } break; } default: @@ -3447,7 +3461,7 @@ collect_fences: xe_assert(vm->xe, current_fence == n_fence); dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1), - 1, false); + 1); fence = &cf->base; for_each_tile(tile, vm->xe, id) { @@ -4239,7 +4253,7 @@ struct dma_fence *xe_vm_bind_kernel_bo(struct xe_vm *vm, struct xe_bo *bo, ops = vm_bind_ioctl_ops_create(vm, &vops, bo, 0, addr, xe_bo_size(bo), DRM_XE_VM_BIND_OP_MAP, 0, 0, - vm->xe->pat.idx[cache_lvl]); + xe_cache_pat_idx(vm->xe, cache_lvl)); if (IS_ERR(ops)) { err = PTR_ERR(ops); goto release_vm_lock; @@ -4442,7 +4456,7 @@ struct xe_vm_snapshot { #define XE_VM_SNAP_FLAG_IS_NULL BIT(2) unsigned long flags; int uapi_mem_region; - int pat_index; + u16 pat_index; int cpu_caching; struct xe_bo *bo; void *data; diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index c4fb29004195..246fe1843142 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -643,7 +643,7 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil xe_device_is_l2_flush_optimized(xe) && (pat_index != 19 && coh_mode != XE_COH_2WAY))) { err = -EINVAL; - goto madv_fini; + goto free_vmas; } } diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index a94827d7fbec..635ed29b9a69 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -412,10 +412,11 @@ struct xe_vm { struct xe_vma_op_map { /** @vma: VMA to map */ struct xe_vma *vma; + /** @vma_flags: VMA flags for this operation */ unsigned int vma_flags; /** @immediate: Immediate bind */ bool immediate; - /** @read_only: Read only */ + /** @invalidate_on_bind: Invalidate on bind */ bool invalidate_on_bind; /** @request_decompress: schedule decompression for GPU map */ bool request_decompress; diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 0538dcb8b18c..23eb7edbdd57 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -13,6 +13,7 @@ #include "regs/xe_gt_regs.h" #include "regs/xe_regs.h" #include "xe_assert.h" +#include "xe_bo.h" #include "xe_device.h" #include "xe_force_wake.h" #include "xe_gt_mcr.h" @@ -250,6 +251,27 @@ static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram, } /** + * xe_map_resource_to_region - Map ttm resource to vram memory region + * @res: The ttm resource + * + * Get vram memory region using vram memory manager managing this resource + * + * Returns: pointer to xe_vram_region + */ +struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res) +{ + struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); + struct ttm_resource_manager *mgr; + struct xe_ttm_vram_mgr *vram_mgr; + + xe_assert(xe, mem_type_is_vram(res->mem_type)); + mgr = ttm_manager_type(&xe->ttm, res->mem_type); + vram_mgr = to_xe_ttm_vram_mgr(mgr); + + return container_of(vram_mgr, struct xe_vram_region, ttm); +} + +/** * xe_vram_probe() - Probe VRAM configuration * @xe: the &xe_device * diff --git a/drivers/gpu/drm/xe/xe_vram.h b/drivers/gpu/drm/xe/xe_vram.h index 72860f714fc6..dd1c8bf17922 100644 --- a/drivers/gpu/drm/xe/xe_vram.h +++ b/drivers/gpu/drm/xe/xe_vram.h @@ -10,7 +10,9 @@ struct xe_device; struct xe_vram_region; +struct ttm_resource; +struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res); int xe_vram_probe(struct xe_device *xe); struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement); diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 33df43d0bede..b9d9fe0801aa 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -130,7 +130,7 @@ __diag_push(); __diag_ignore_all("-Woverride-init", "Allow field overrides in table"); -static const struct xe_rtp_entry_sr gt_was[] = { +static const struct xe_rtp_table_sr gt_was = XE_RTP_TABLE_SR( /* Workarounds applying over a range of IPs */ { XE_RTP_NAME("14011060649"), @@ -306,9 +306,9 @@ static const struct xe_rtp_entry_sr gt_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)), XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES)) }, -}; +); -static const struct xe_rtp_entry_sr engine_was[] = { +static const struct xe_rtp_table_sr engine_was = XE_RTP_TABLE_SR( /* Workarounds applying over a range of IPs */ { XE_RTP_NAME("22010931296, 18011464164, 14010919138"), @@ -601,9 +601,22 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(ROW_CHICKEN5, CPSS_AWARE_DIS)) }, -}; -static const struct xe_rtp_entry_sr lrc_was[] = { + /* Xe3p_XPC */ + + { XE_RTP_NAME("14026999295"), + XE_RTP_RULES(GRAPHICS_VERSION(3511), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN3, DIS_EU_GRF_POISON_TO_LSC)) + }, + { XE_RTP_NAME("18044193044"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_CHICKEN, BIT_APQ_OPT_DIS)) + }, +); + +static const struct xe_rtp_table_sr lrc_was = XE_RTP_TABLE_SR( { XE_RTP_NAME("16011163337"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), /* read verification is ignored due to 1608008084. */ @@ -781,21 +794,29 @@ static const struct xe_rtp_entry_sr lrc_was[] = { ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX)) }, -}; +); -static __maybe_unused const struct xe_rtp_entry oob_was[] = { +static const struct xe_rtp_entry oob_was_entries[] = { #include <generated/xe_wa_oob.c> - {} }; -static_assert(ARRAY_SIZE(oob_was) - 1 == _XE_WA_OOB_COUNT); +static_assert(ARRAY_SIZE(oob_was_entries) == _XE_WA_OOB_COUNT); -static __maybe_unused const struct xe_rtp_entry device_oob_was[] = { +static __maybe_unused const struct xe_rtp_table oob_was = { + .entries = oob_was_entries, + .n_entries = ARRAY_SIZE(oob_was_entries), +}; + +static const struct xe_rtp_entry device_oob_was_entries[] = { #include <generated/xe_device_wa_oob.c> - {} }; -static_assert(ARRAY_SIZE(device_oob_was) - 1 == _XE_DEVICE_WA_OOB_COUNT); +static_assert(ARRAY_SIZE(device_oob_was_entries) == _XE_DEVICE_WA_OOB_COUNT); + +static __maybe_unused const struct xe_rtp_table device_oob_was = { + .entries = device_oob_was_entries, + .n_entries = ARRAY_SIZE(device_oob_was_entries), +}; __diag_pop(); @@ -811,10 +832,10 @@ void xe_wa_process_device_oob(struct xe_device *xe) { struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(xe); - xe_rtp_process_ctx_enable_active_tracking(&ctx, xe->wa_active.oob, ARRAY_SIZE(device_oob_was)); + xe_rtp_process_ctx_enable_active_tracking(&ctx, xe->wa_active.oob, device_oob_was.n_entries); xe->wa_active.oob_initialized = true; - xe_rtp_process(&ctx, device_oob_was); + xe_rtp_process(&ctx, &device_oob_was); } /** @@ -829,9 +850,9 @@ void xe_wa_process_gt_oob(struct xe_gt *gt) struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->wa_active.oob, - ARRAY_SIZE(oob_was)); + oob_was.n_entries); gt->wa_active.oob_initialized = true; - xe_rtp_process(&ctx, oob_was); + xe_rtp_process(&ctx, &oob_was); } /** @@ -846,9 +867,8 @@ void xe_wa_process_gt(struct xe_gt *gt) struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt); xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->wa_active.gt, - ARRAY_SIZE(gt_was)); - xe_rtp_process_to_sr(&ctx, gt_was, ARRAY_SIZE(gt_was), - >->reg_sr, false); + gt_was.n_entries); + xe_rtp_process_to_sr(&ctx, >_was, >->reg_sr, false); } EXPORT_SYMBOL_IF_KUNIT(xe_wa_process_gt); @@ -865,9 +885,8 @@ void xe_wa_process_engine(struct xe_hw_engine *hwe) struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->wa_active.engine, - ARRAY_SIZE(engine_was)); - xe_rtp_process_to_sr(&ctx, engine_was, ARRAY_SIZE(engine_was), - &hwe->reg_sr, false); + engine_was.n_entries); + xe_rtp_process_to_sr(&ctx, &engine_was, &hwe->reg_sr, false); } /** @@ -883,9 +902,8 @@ void xe_wa_process_lrc(struct xe_hw_engine *hwe) struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); xe_rtp_process_ctx_enable_active_tracking(&ctx, hwe->gt->wa_active.lrc, - ARRAY_SIZE(lrc_was)); - xe_rtp_process_to_sr(&ctx, lrc_was, ARRAY_SIZE(lrc_was), - &hwe->reg_lrc, true); + lrc_was.n_entries); + xe_rtp_process_to_sr(&ctx, &lrc_was, &hwe->reg_lrc, true); } /** @@ -899,7 +917,7 @@ int xe_wa_device_init(struct xe_device *xe) unsigned long *p; p = drmm_kzalloc(&xe->drm, - sizeof(*p) * BITS_TO_LONGS(ARRAY_SIZE(device_oob_was)), + sizeof(*p) * BITS_TO_LONGS(device_oob_was.n_entries), GFP_KERNEL); if (!p) @@ -922,10 +940,10 @@ int xe_wa_gt_init(struct xe_gt *gt) size_t n_oob, n_lrc, n_engine, n_gt, total; unsigned long *p; - n_gt = BITS_TO_LONGS(ARRAY_SIZE(gt_was)); - n_engine = BITS_TO_LONGS(ARRAY_SIZE(engine_was)); - n_lrc = BITS_TO_LONGS(ARRAY_SIZE(lrc_was)); - n_oob = BITS_TO_LONGS(ARRAY_SIZE(oob_was)); + n_gt = BITS_TO_LONGS(gt_was.n_entries); + n_engine = BITS_TO_LONGS(engine_was.n_entries); + n_lrc = BITS_TO_LONGS(lrc_was.n_entries); + n_oob = BITS_TO_LONGS(oob_was.n_entries); total = n_gt + n_engine + n_lrc + n_oob; p = drmm_kzalloc(&xe->drm, sizeof(*p) * total, GFP_KERNEL); @@ -949,9 +967,9 @@ void xe_wa_device_dump(struct xe_device *xe, struct drm_printer *p) size_t idx; drm_printf(p, "Device OOB Workarounds\n"); - for_each_set_bit(idx, xe->wa_active.oob, ARRAY_SIZE(device_oob_was)) - if (device_oob_was[idx].name) - drm_printf_indent(p, 1, "%s\n", device_oob_was[idx].name); + for_each_set_bit(idx, xe->wa_active.oob, device_oob_was.n_entries) + if (device_oob_was.entries[idx].name) + drm_printf_indent(p, 1, "%s\n", device_oob_was.entries[idx].name); } /** @@ -966,24 +984,24 @@ int xe_wa_gt_dump(struct xe_gt *gt, struct drm_printer *p) size_t idx; drm_printf(p, "GT Workarounds\n"); - for_each_set_bit(idx, gt->wa_active.gt, ARRAY_SIZE(gt_was)) - drm_printf_indent(p, 1, "%s\n", gt_was[idx].name); + for_each_set_bit(idx, gt->wa_active.gt, gt_was.n_entries) + drm_printf_indent(p, 1, "%s\n", gt_was.entries[idx].name); drm_puts(p, "\n"); drm_printf(p, "Engine Workarounds\n"); - for_each_set_bit(idx, gt->wa_active.engine, ARRAY_SIZE(engine_was)) - drm_printf_indent(p, 1, "%s\n", engine_was[idx].name); + for_each_set_bit(idx, gt->wa_active.engine, engine_was.n_entries) + drm_printf_indent(p, 1, "%s\n", engine_was.entries[idx].name); drm_puts(p, "\n"); drm_printf(p, "LRC Workarounds\n"); - for_each_set_bit(idx, gt->wa_active.lrc, ARRAY_SIZE(lrc_was)) - drm_printf_indent(p, 1, "%s\n", lrc_was[idx].name); + for_each_set_bit(idx, gt->wa_active.lrc, lrc_was.n_entries) + drm_printf_indent(p, 1, "%s\n", lrc_was.entries[idx].name); drm_puts(p, "\n"); drm_printf(p, "OOB Workarounds\n"); - for_each_set_bit(idx, gt->wa_active.oob, ARRAY_SIZE(oob_was)) - if (oob_was[idx].name) - drm_printf_indent(p, 1, "%s\n", oob_was[idx].name); + for_each_set_bit(idx, gt->wa_active.oob, oob_was.n_entries) + if (oob_was.entries[idx].name) + drm_printf_indent(p, 1, "%s\n", oob_was.entries[idx].name); return 0; } diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h index 8fd6a5af0910..a5f7d33c1b32 100644 --- a/drivers/gpu/drm/xe/xe_wa.h +++ b/drivers/gpu/drm/xe/xe_wa.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_WA_ -#define _XE_WA_ +#ifndef _XE_WA_H_ +#define _XE_WA_H_ #include "xe_assert.h" diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index 80b54b195f20..f8a185103b80 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -51,6 +51,13 @@ MEDIA_VERSION_RANGE(2000, 3002), FUNC(xe_rtp_match_psmi_enabled) 16023683509 MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled) MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), FUNC(xe_rtp_match_psmi_enabled) +14025515070 GRAPHICS_VERSION(2004) + MEDIA_VERSION_RANGE(1301, 3000) + MEDIA_VERSION(3002) + GRAPHICS_VERSION_RANGE(3000, 3001) + GRAPHICS_VERSION_RANGE(3003, 3005) + MEDIA_VERSION(3500) + GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0) 15015404425_disable PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER) 16026007364 MEDIA_VERSION(3000) diff --git a/drivers/gpu/drm/xe/xe_wopcm.c b/drivers/gpu/drm/xe/xe_wopcm.c index 900daf1d1b1b..fe65ed246775 100644 --- a/drivers/gpu/drm/xe/xe_wopcm.c +++ b/drivers/gpu/drm/xe/xe_wopcm.c @@ -49,9 +49,9 @@ */ /* Default WOPCM size is 2MB from Gen11, 1MB on previous platforms */ -/* FIXME: Larger size require for 2 tile PVC, do a proper probe sooner or later */ +/* FIXME: Larger size require for some platforms, do a proper probe sooner or later */ #define DGFX_WOPCM_SIZE SZ_4M -/* FIXME: Larger size require for MTL, do a proper probe sooner or later */ +#define LNL_WOPCM_SIZE SZ_8M #define MTL_WOPCM_SIZE SZ_4M #define WOPCM_SIZE SZ_2M @@ -179,9 +179,14 @@ err_out: u32 xe_wopcm_size(struct xe_device *xe) { - return IS_DGFX(xe) ? DGFX_WOPCM_SIZE : - xe->info.platform == XE_METEORLAKE ? MTL_WOPCM_SIZE : - WOPCM_SIZE; + if (xe->info.platform >= XE_LUNARLAKE) + return LNL_WOPCM_SIZE; + else if (IS_DGFX(xe)) + return DGFX_WOPCM_SIZE; + else if (xe->info.platform == XE_METEORLAKE) + return MTL_WOPCM_SIZE; + else + return WOPCM_SIZE; } static u32 max_wopcm_size(struct xe_device *xe) |