diff options
Diffstat (limited to 'drivers/gpu/drm/xe')
179 files changed, 4328 insertions, 1572 deletions
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 49de1c22a469..09661f079d03 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -88,6 +88,7 @@ xe-y += xe_bb.o \ xe_irq.o \ xe_late_bind_fw.o \ xe_lrc.o \ + xe_mem_pool.o \ xe_migrate.o \ xe_mmio.o \ xe_mmio_gem.o \ @@ -112,6 +113,7 @@ xe-y += xe_bb.o \ xe_pxp_submit.o \ xe_query.o \ xe_range_fence.o \ + xe_ras.o \ xe_reg_sr.o \ xe_reg_whitelist.o \ xe_ring_ops.o \ @@ -123,6 +125,9 @@ xe-y += xe_bb.o \ xe_step.o \ xe_survivability_mode.o \ xe_sync.o \ + xe_sysctrl.o \ + xe_sysctrl_event.o \ + xe_sysctrl_mailbox.o \ xe_tile.o \ xe_tile_sysfs.o \ xe_tlb_inval.o \ @@ -211,7 +216,6 @@ $(obj)/i915-display/%.o: $(srctree)/drivers/gpu/drm/i915/display/%.c FORCE # Display code specific to xe xe-$(CONFIG_DRM_XE_DISPLAY) += \ - display/intel_fbdev_fb.o \ display/xe_display.o \ display/xe_display_bo.o \ display/xe_display_pcode.o \ @@ -261,6 +265,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/intel_display_power.o \ i915-display/intel_display_power_map.o \ i915-display/intel_display_power_well.o \ + i915-display/intel_display_reset.o \ i915-display/intel_display_rpm.o \ i915-display/intel_display_rps.o \ i915-display/intel_display_trace.o \ @@ -304,6 +309,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/intel_link_bw.o \ i915-display/intel_lspcon.o \ i915-display/intel_lt_phy.o \ + i915-display/intel_mchbar.o \ i915-display/intel_modeset_lock.o \ i915-display/intel_modeset_setup.o \ i915-display/intel_modeset_verify.o \ @@ -330,7 +336,8 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/skl_prefill.o \ i915-display/skl_scaler.o \ i915-display/skl_universal_plane.o \ - i915-display/skl_watermark.o + i915-display/skl_watermark.o \ + i915-display/vlv_sideband.o ifeq ($(CONFIG_ACPI),y) xe-$(CONFIG_DRM_XE_DISPLAY) += \ diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h index e33bd622ab44..644f5a4226d7 100644 --- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h @@ -192,6 +192,13 @@ enum { * `GuC KLV`_ keys available for use with PF2GUC_UPDATE_VGT_POLICY. * * _`GUC_KLV_VGT_POLICY_SCHED_IF_IDLE` : 0x8001 + * [From 70.12.0] + * This config allows to update scheduling priority of PF and all VFs at once. + * Setting this policy to 0 updates all VFs scheduling priorities to LOW, and + * setting this policy to 1 updates all VFs scheduling priorities to NORMAL. + * Those changes will take effect on the next VF-Switch event. + * + * [Before 70.12.0] * This config sets whether strict scheduling is enabled whereby any VF * that doesn’t have work to submit is still allocated a fixed execution * time-slice to ensure active VFs execution is always consistent even @@ -496,6 +503,7 @@ enum xe_guc_klv_ids { GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO = 0x900a, GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH = 0x900b, GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG = 0x900c, + GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG = 0x900e, }; #endif diff --git a/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h b/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h new file mode 100644 index 000000000000..4cbde267ac44 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_ABI_H_ +#define _XE_SYSCTRL_ABI_H_ + +#include <linux/types.h> + +/** + * DOC: System Controller ABI + * + * This header defines the Application Binary Interface (ABI) used by + * drm/xe to communicate with System Controller firmware on Intel Xe3p + * discrete GPU platforms. + * + * System Controller (sysctrl) is a firmware-managed entity on Intel + * dGPUs responsible for certain low-level platform management + * functions. + * + * Communication protocol: + * + * Communication uses a mailbox interface with messages composed of: + * + * - Application message header (struct xe_sysctrl_app_msg_hdr) + * containing group_id, command, and version + * - Variable-length, command-specific payload + * + * Message header format: + * + * The 32-bit application message header is packed as: + * + * - Bits [7:0] : Group ID identifying command group + * - Bits [15:8] : Command identifier within group + * - Bits [23:16] : Command version for interface compatibility + * - Bits [31:24] : Reserved, must be zero + * + * This header defines firmware ABI message formats and constants shared + * between driver and System Controller firmware. + */ + +/** + * struct xe_sysctrl_app_msg_hdr - Application layer message header + * @data: 32-bit header data + * + * Header structure for application-level messages. + */ +struct xe_sysctrl_app_msg_hdr { + u32 data; +} __packed; + +#define SYSCTRL_HDR_GROUP_ID_MASK GENMASK(7, 0) +#define SYSCTRL_HDR_COMMAND_MASK GENMASK(14, 8) +#define SYSCTRL_HDR_COMMAND_MAX 0x7f +#define SYSCTRL_HDR_IS_RESPONSE BIT(15) +#define SYSCTRL_HDR_RESERVED_MASK GENMASK(23, 16) +#define SYSCTRL_HDR_RESULT_MASK GENMASK(31, 24) + +#define APP_HDR_GROUP_ID_MASK GENMASK(7, 0) +#define APP_HDR_COMMAND_MASK GENMASK(15, 8) +#define APP_HDR_VERSION_MASK GENMASK(23, 16) +#define APP_HDR_RESERVED_MASK GENMASK(31, 24) + +#endif diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h index ce986f0e8f38..552975a30ba2 100644 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h +++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h @@ -3,4 +3,12 @@ * Copyright © 2023 Intel Corporation */ -#include "../../i915/intel_clock_gating.h" +#ifndef __INTEL_CLOCK_GATING_H__ +#define __INTEL_CLOCK_GATING_H__ + +struct drm_device; + +static inline void intel_clock_gating_init(struct drm_device *drm) {} +static inline void intel_clock_gating_hooks_init(struct drm_device *drm) {} + +#endif /* __INTEL_CLOCK_GATING_H__ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h deleted file mode 100644 index 55b316985340..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h +++ /dev/null @@ -1,6 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#include "../../i915/intel_mchbar_regs.h" diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h deleted file mode 100644 index 8c15867fd613..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h +++ /dev/null @@ -1,6 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#include "../../i915/intel_pci_config.h" diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h deleted file mode 100644 index 0eabe2866f5f..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#ifndef __INTEL_STEP_H__ -#define __INTEL_STEP_H__ - -#include "xe_step_types.h" - -#define intel_step xe_step - -#endif /* __INTEL_STEP_H__ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h b/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h deleted file mode 100644 index 69e1935e9cdf..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2013-2021 Intel Corporation - */ - -#ifndef _VLV_IOSF_SB_H_ -#define _VLV_IOSF_SB_H_ - -#include <linux/types.h> - -#include "vlv_iosf_sb_reg.h" - -struct drm_device; - -enum vlv_iosf_sb_unit { - VLV_IOSF_SB_BUNIT, - VLV_IOSF_SB_CCK, - VLV_IOSF_SB_CCU, - VLV_IOSF_SB_DPIO, - VLV_IOSF_SB_DPIO_2, - VLV_IOSF_SB_FLISDSI, - VLV_IOSF_SB_GPIO, - VLV_IOSF_SB_NC, - VLV_IOSF_SB_PUNIT, -}; - -static inline void vlv_iosf_sb_get(struct drm_device *drm, unsigned long ports) -{ -} -static inline u32 vlv_iosf_sb_read(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr) -{ - return 0; -} -static inline int vlv_iosf_sb_write(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr, u32 val) -{ - return 0; -} -static inline void vlv_iosf_sb_put(struct drm_device *drm, unsigned long ports) -{ -} - -#endif /* _VLV_IOSF_SB_H_ */ diff --git a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h b/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h deleted file mode 100644 index cb7fa8e794a6..000000000000 --- a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h +++ /dev/null @@ -1,6 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#include "../../i915/vlv_iosf_sb_reg.h" diff --git a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c deleted file mode 100644 index d7030e4d814c..000000000000 --- a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c +++ /dev/null @@ -1,120 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#include <linux/fb.h> - -#include "intel_fbdev_fb.h" -#include "xe_bo.h" -#include "xe_ttm_stolen_mgr.h" -#include "xe_wa.h" - -#include <generated/xe_device_wa_oob.h> - -/* - * FIXME: There shouldn't be any reason to have XE_PAGE_SIZE stride - * alignment. The same 64 as i915 uses should be fine, and we shouldn't need to - * have driver specific values. However, dropping the stride alignment to 64 - * leads to underflowing the bo pin count in the atomic cleanup work. - */ -u32 intel_fbdev_fb_pitch_align(u32 stride) -{ - return ALIGN(stride, XE_PAGE_SIZE); -} - -bool intel_fbdev_fb_prefer_stolen(struct drm_device *drm, unsigned int size) -{ - struct xe_device *xe = to_xe_device(drm); - struct ttm_resource_manager *stolen; - - stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); - if (!stolen) - return false; - - if (IS_DGFX(xe)) - return false; - - if (XE_DEVICE_WA(xe, 22019338487_display)) - return false; - - /* - * If the FB is too big, just don't use it since fbdev is not very - * important and we should probably use that space with FBC or other - * features. - */ - return stolen->size >= size * 2; -} - -struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size) -{ - struct xe_device *xe = to_xe_device(drm); - struct xe_bo *obj; - - obj = ERR_PTR(-ENODEV); - - if (intel_fbdev_fb_prefer_stolen(drm, size)) { - obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), - size, - ttm_bo_type_kernel, - XE_BO_FLAG_FORCE_WC | - XE_BO_FLAG_STOLEN | - XE_BO_FLAG_GGTT, - false); - if (!IS_ERR(obj)) - drm_info(&xe->drm, "Allocated fbdev into stolen\n"); - else - drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj)); - } else { - drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n"); - } - - if (IS_ERR(obj)) { - obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size, - ttm_bo_type_kernel, - XE_BO_FLAG_FORCE_WC | - XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | - XE_BO_FLAG_GGTT, - false); - } - - if (IS_ERR(obj)) { - drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj); - return ERR_PTR(-ENOMEM); - } - - return &obj->ttm.base; -} - -void intel_fbdev_fb_bo_destroy(struct drm_gem_object *obj) -{ - xe_bo_unpin_map_no_vm(gem_to_xe_bo(obj)); -} - -int intel_fbdev_fb_fill_info(struct drm_device *drm, struct fb_info *info, - struct drm_gem_object *_obj, struct i915_vma *vma) -{ - struct xe_bo *obj = gem_to_xe_bo(_obj); - struct pci_dev *pdev = to_pci_dev(drm->dev); - - if (!(obj->flags & XE_BO_FLAG_SYSTEM)) { - if (obj->flags & XE_BO_FLAG_STOLEN) - info->fix.smem_start = xe_ttm_stolen_io_offset(obj, 0); - else - info->fix.smem_start = - pci_resource_start(pdev, 2) + - xe_bo_addr(obj, 0, XE_PAGE_SIZE); - - info->fix.smem_len = obj->ttm.base.size; - } else { - /* XXX: Pure fiction, as the BO may not be physically accessible.. */ - info->fix.smem_start = 0; - info->fix.smem_len = obj->ttm.base.size; - } - XE_WARN_ON(iosys_map_is_null(&obj->vmap)); - - info->screen_base = obj->vmap.vaddr_iomem; - info->screen_size = obj->ttm.base.size; - - return 0; -} diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c index 00dfa68af29a..053abd6f6514 100644 --- a/drivers/gpu/drm/xe/display/xe_display.c +++ b/drivers/gpu/drm/xe/display/xe_display.c @@ -36,10 +36,12 @@ #include "intel_hotplug.h" #include "intel_opregion.h" #include "skl_watermark.h" +#include "xe_device.h" #include "xe_display_bo.h" #include "xe_display_pcode.h" #include "xe_display_rpm.h" #include "xe_dsb_buffer.h" +#include "xe_fb_pin.h" #include "xe_frontbuffer.h" #include "xe_hdcp_gsc.h" #include "xe_initial_plane.h" @@ -69,31 +71,9 @@ bool xe_display_driver_probe_defer(struct pci_dev *pdev) return intel_display_driver_probe_defer(pdev); } -/** - * xe_display_driver_set_hooks - Add driver flags and hooks for display - * @driver: DRM device driver - * - * Set features and function hooks in @driver that are needed for driving the - * display IP. This sets the driver's capability of driving display, regardless - * if the device has it enabled - * - * Note: This is called before xe or display device creation. - */ -void xe_display_driver_set_hooks(struct drm_driver *driver) -{ - if (!xe_modparam.probe_display) - return; - -#ifdef CONFIG_DRM_FBDEV_EMULATION - driver->fbdev_probe = intel_fbdev_driver_fbdev_probe; -#endif - - driver->driver_features |= DRIVER_MODESET | DRIVER_ATOMIC; -} - static void unset_display_features(struct xe_device *xe) { - xe->drm.driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC); + xe->drm.driver_features &= ~XE_DISPLAY_DRIVER_FEATURES; } static void xe_display_fini_early(void *arg) @@ -520,9 +500,10 @@ void xe_display_pm_runtime_resume(struct xe_device *xe) static void display_device_remove(struct drm_device *dev, void *arg) { - struct intel_display *display = arg; + struct xe_device *xe = arg; - intel_display_device_remove(display); + intel_display_device_remove(xe->display); + xe->display = NULL; } static bool irq_enabled(struct drm_device *drm) @@ -552,6 +533,7 @@ static bool has_auxccs(struct drm_device *drm) static const struct intel_display_parent_interface parent = { .bo = &xe_display_bo_interface, .dsb = &xe_display_dsb_interface, + .fb_pin = &xe_display_fb_pin_interface, .frontbuffer = &xe_display_frontbuffer_interface, .hdcp = &xe_display_hdcp_interface, .initial_plane = &xe_display_initial_plane_interface, @@ -587,12 +569,12 @@ int xe_display_probe(struct xe_device *xe) if (IS_ERR(display)) return PTR_ERR(display); - err = drmm_add_action_or_reset(&xe->drm, display_device_remove, display); + xe->display = display; + + err = drmm_add_action_or_reset(&xe->drm, display_device_remove, xe); if (err) return err; - xe->display = display; - if (intel_display_device_present(display)) return 0; @@ -601,3 +583,11 @@ no_display: unset_display_features(xe); return 0; } + +#ifdef CONFIG_DRM_FBDEV_EMULATION +int xe_display_driver_fbdev_probe(struct drm_fb_helper *fbh, + struct drm_fb_helper_surface_size *sizes) +{ + return intel_fbdev_driver_fbdev_probe(fbh, sizes); +} +#endif diff --git a/drivers/gpu/drm/xe/display/xe_display.h b/drivers/gpu/drm/xe/display/xe_display.h index 76db95c25f7e..60291cb154df 100644 --- a/drivers/gpu/drm/xe/display/xe_display.h +++ b/drivers/gpu/drm/xe/display/xe_display.h @@ -6,14 +6,20 @@ #ifndef _XE_DISPLAY_H_ #define _XE_DISPLAY_H_ -#include "xe_device.h" +#include <linux/types.h> struct drm_driver; +struct drm_fb_helper; +struct drm_fb_helper_surface_size; +struct pci_dev; +struct xe_device; #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) bool xe_display_driver_probe_defer(struct pci_dev *pdev); -void xe_display_driver_set_hooks(struct drm_driver *driver); + +int xe_display_driver_fbdev_probe(struct drm_fb_helper *fbh, + struct drm_fb_helper_surface_size *sizes); int xe_display_probe(struct xe_device *xe); @@ -38,11 +44,18 @@ void xe_display_pm_runtime_suspend(struct xe_device *xe); void xe_display_pm_runtime_suspend_late(struct xe_device *xe); void xe_display_pm_runtime_resume(struct xe_device *xe); +#define XE_DISPLAY_DRIVER_FEATURES (DRIVER_MODESET | DRIVER_ATOMIC) +#define XE_DISPLAY_DRIVER_OPS \ + .fbdev_probe = PTR_IF(IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION), \ + xe_display_driver_fbdev_probe) + #else +#define XE_DISPLAY_DRIVER_FEATURES 0 +#define XE_DISPLAY_DRIVER_OPS \ + .fbdev_probe = NULL + static inline int xe_display_driver_probe_defer(struct pci_dev *pdev) { return 0; } -static inline void xe_display_driver_set_hooks(struct drm_driver *driver) { } -static inline void xe_display_driver_remove(struct xe_device *xe) {} static inline int xe_display_probe(struct xe_device *xe) { return 0; } diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.c b/drivers/gpu/drm/xe/display/xe_display_bo.c index 1d81b9908265..7fbac223b097 100644 --- a/drivers/gpu/drm/xe/display/xe_display_bo.c +++ b/drivers/gpu/drm/xe/display/xe_display_bo.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: MIT /* Copyright © 2024 Intel Corporation */ +#include <linux/fb.h> + #include <drm/drm_gem.h> #include <drm/intel/display_parent_interface.h> @@ -8,6 +10,10 @@ #include "xe_bo.h" #include "xe_display_bo.h" #include "xe_pxp.h" +#include "xe_ttm_stolen_mgr.h" +#include "xe_wa.h" + +#include <generated/xe_device_wa_oob.h> static bool xe_display_bo_is_protected(struct drm_gem_object *obj) { @@ -101,6 +107,114 @@ xe_display_bo_framebuffer_lookup(struct drm_device *drm, return gem; } +#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) +/* + * FIXME: There shouldn't be any reason to have XE_PAGE_SIZE stride + * alignment. The same 64 as i915 uses should be fine, and we shouldn't need to + * have driver specific values. However, dropping the stride alignment to 64 + * leads to underflowing the bo pin count in the atomic cleanup work. + */ +static u32 xe_display_bo_fbdev_pitch_align(u32 stride) +{ + return ALIGN(stride, XE_PAGE_SIZE); +} + +bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size) +{ + struct ttm_resource_manager *stolen; + + stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); + if (!stolen) + return false; + + if (IS_DGFX(xe)) + return false; + + if (XE_DEVICE_WA(xe, 22019338487_display)) + return false; + + /* + * If the FB is too big, just don't use it since fbdev is not very + * important and we should probably use that space with FBC or other + * features. + */ + return stolen->size >= (size * 2) >> PAGE_SHIFT; +} + +static struct drm_gem_object *xe_display_bo_fbdev_create(struct drm_device *drm, int size) +{ + struct xe_device *xe = to_xe_device(drm); + struct xe_bo *obj; + + obj = ERR_PTR(-ENODEV); + + if (xe_display_bo_fbdev_prefer_stolen(xe, size)) { + obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), + size, + ttm_bo_type_kernel, + XE_BO_FLAG_FORCE_WC | + XE_BO_FLAG_STOLEN | + XE_BO_FLAG_GGTT, + false); + if (!IS_ERR(obj)) + drm_info(&xe->drm, "Allocated fbdev into stolen\n"); + else + drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj)); + } else { + drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n"); + } + + if (IS_ERR(obj)) { + obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size, + ttm_bo_type_kernel, + XE_BO_FLAG_FORCE_WC | + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | + XE_BO_FLAG_GGTT, + false); + } + + if (IS_ERR(obj)) { + drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj); + return ERR_PTR(-ENOMEM); + } + + return &obj->ttm.base; +} + +static void xe_display_bo_fbdev_destroy(struct drm_gem_object *obj) +{ + xe_bo_unpin_map_no_vm(gem_to_xe_bo(obj)); +} + +static int xe_display_bo_fbdev_fill_info(struct drm_gem_object *_obj, struct fb_info *info, + struct i915_vma *vma) +{ + struct xe_bo *obj = gem_to_xe_bo(_obj); + struct pci_dev *pdev = to_pci_dev(_obj->dev->dev); + + if (!(obj->flags & XE_BO_FLAG_SYSTEM)) { + if (obj->flags & XE_BO_FLAG_STOLEN) + info->fix.smem_start = xe_ttm_stolen_io_offset(obj, 0); + else + info->fix.smem_start = + pci_resource_start(pdev, 2) + + xe_bo_addr(obj, 0, XE_PAGE_SIZE); + + info->fix.smem_len = obj->ttm.base.size; + } else { + /* XXX: Pure fiction, as the BO may not be physically accessible.. */ + info->fix.smem_start = 0; + info->fix.smem_len = obj->ttm.base.size; + } + XE_WARN_ON(iosys_map_is_null(&obj->vmap)); + + info->screen_base = obj->vmap.vaddr_iomem; + info->screen_size = obj->ttm.base.size; + + return 0; +} +#endif + const struct intel_display_bo_interface xe_display_bo_interface = { .is_protected = xe_display_bo_is_protected, .key_check = xe_pxp_obj_key_check, @@ -109,4 +223,10 @@ const struct intel_display_bo_interface xe_display_bo_interface = { .framebuffer_init = xe_display_bo_framebuffer_init, .framebuffer_fini = xe_display_bo_framebuffer_fini, .framebuffer_lookup = xe_display_bo_framebuffer_lookup, +#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) + .fbdev_create = xe_display_bo_fbdev_create, + .fbdev_destroy = xe_display_bo_fbdev_destroy, + .fbdev_fill_info = xe_display_bo_fbdev_fill_info, + .fbdev_pitch_align = xe_display_bo_fbdev_pitch_align, +#endif }; diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.h b/drivers/gpu/drm/xe/display/xe_display_bo.h index 6879c104b0b1..c72056884ff4 100644 --- a/drivers/gpu/drm/xe/display/xe_display_bo.h +++ b/drivers/gpu/drm/xe/display/xe_display_bo.h @@ -4,6 +4,12 @@ #ifndef __XE_DISPLAY_BO_H__ #define __XE_DISPLAY_BO_H__ +#include <linux/types.h> + +struct xe_device; + +bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size); + extern const struct intel_display_bo_interface xe_display_bo_interface; #endif diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index e45a1e7a4670..f93c98bec5b5 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -3,17 +3,21 @@ * Copyright © 2021 Intel Corporation */ +#include <drm/intel/display_parent_interface.h> #include <drm/ttm/ttm_bo.h> -#include "intel_display_core.h" -#include "intel_display_types.h" +/* FIXME move the types to parent interface? */ +#include "i915_gtt_view_types.h" + +/* FIXME move intel_remapped_info_size() & co. to parent interface? */ #include "intel_fb.h" -#include "intel_fb_pin.h" -#include "intel_fbdev.h" + #include "xe_bo.h" #include "xe_device.h" #include "xe_display_vma.h" +#include "xe_fb_pin.h" #include "xe_ggtt.h" +#include "xe_pat.h" #include "xe_pm.h" #include "xe_vram_types.h" @@ -24,7 +28,7 @@ write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ struct xe_device *xe = xe_bo_device(bo); struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; u32 column, row; - u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); + u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, * by writing dpt/ggtt in a different order? @@ -64,7 +68,7 @@ write_dpt_remapped_linear(struct xe_bo *bo, struct iosys_map *map, struct xe_device *xe = xe_bo_device(bo); struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, - xe->pat.idx[XE_CACHE_NONE]); + xe_cache_pat_idx(xe, XE_CACHE_NONE)); unsigned int offset = plane->offset * XE_PAGE_SIZE; unsigned int size = plane->size; @@ -87,7 +91,7 @@ write_dpt_remapped_tiled(struct xe_bo *bo, struct iosys_map *map, struct xe_device *xe = xe_bo_device(bo); struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, - xe->pat.idx[XE_CACHE_NONE]); + xe_cache_pat_idx(xe, XE_CACHE_NONE)); unsigned int offset, column, row; for (row = 0; row < plane->height; row++) { @@ -139,22 +143,21 @@ write_dpt_remapped(struct xe_bo *bo, } } -static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, - const struct i915_gtt_view *view, - struct i915_vma *vma, - unsigned int alignment) +static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma *vma) { - struct xe_device *xe = to_xe_device(fb->base.dev); + struct xe_device *xe = to_xe_device(obj->dev); struct xe_tile *tile0 = xe_device_get_root_tile(xe); struct xe_ggtt *ggtt = tile0->mem.ggtt; - struct drm_gem_object *obj = intel_fb_bo(&fb->base); + const struct i915_gtt_view *view = pin_params->view; struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; u32 dpt_size, size = bo->ttm.base.size; if (view->type == I915_GTT_VIEW_NORMAL) dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); else if (view->type == I915_GTT_VIEW_REMAPPED) - dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, + dpt_size = ALIGN(intel_remapped_info_size(&view->remapped) * 8, XE_PAGE_SIZE); else /* display uses 4K tiles instead of bytes here, convert to entries.. */ @@ -168,7 +171,7 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, XE_BO_FLAG_VRAM0 | XE_BO_FLAG_GGTT | XE_BO_FLAG_PAGETABLE, - alignment, false); + pin_params->alignment, false); else dpt = xe_bo_create_pin_map_at_novm(xe, tile0, dpt_size, ~0ull, @@ -176,7 +179,7 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, XE_BO_FLAG_STOLEN | XE_BO_FLAG_GGTT | XE_BO_FLAG_PAGETABLE, - alignment, false); + pin_params->alignment, false); if (IS_ERR(dpt)) dpt = xe_bo_create_pin_map_at_novm(xe, tile0, dpt_size, ~0ull, @@ -185,12 +188,12 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, XE_BO_FLAG_GGTT | XE_BO_FLAG_PAGETABLE | XE_BO_FLAG_FORCE_WC, - alignment, false); + pin_params->alignment, false); if (IS_ERR(dpt)) return PTR_ERR(dpt); if (view->type == I915_GTT_VIEW_NORMAL) { - u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); + u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); u32 x; for (x = 0; x < size / XE_PAGE_SIZE; x++) { @@ -268,14 +271,13 @@ static void write_ggtt_rotated_node(struct xe_ggtt *ggtt, struct xe_ggtt_node *n rot_info->plane[i].dst_stride); } -static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, - const struct i915_gtt_view *view, - struct i915_vma *vma, - unsigned int alignment) +static int __xe_pin_fb_vma_ggtt(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma *vma) { - struct drm_gem_object *obj = intel_fb_bo(&fb->base); + const struct i915_gtt_view *view = pin_params->view; struct xe_bo *bo = gem_to_xe_bo(obj); - struct xe_device *xe = to_xe_device(fb->base.dev); + struct xe_device *xe = to_xe_device(obj->dev); struct xe_tile *tile0 = xe_device_get_root_tile(xe); struct xe_ggtt *ggtt = tile0->mem.ggtt; u64 pte, size; @@ -287,7 +289,7 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, */ guard(xe_pm_runtime_noresume)(xe); - align = XE_PAGE_SIZE; + align = max(XE_PAGE_SIZE, pin_params->alignment); if (xe_bo_is_vram(bo) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) align = max(align, SZ_64K); @@ -306,7 +308,7 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, /* display uses tiles instead of bytes here, so convert it back.. */ size = intel_rotation_info_size(&view->rotated) * XE_PAGE_SIZE; - pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); + pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); vma->node = xe_ggtt_insert_node_transform(ggtt, bo, pte, ALIGN(size, align), align, view->type == I915_GTT_VIEW_NORMAL ? @@ -318,25 +320,25 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, return ret; } -static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, - const struct i915_gtt_view *view, - unsigned int alignment) +static struct i915_vma *__xe_pin_fb_vma(struct drm_gem_object *obj, bool is_dpt, + const struct intel_fb_pin_params *pin_params) { - struct drm_device *dev = fb->base.dev; - struct xe_device *xe = to_xe_device(dev); + struct xe_device *xe = to_xe_device(obj->dev); struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); - struct drm_gem_object *obj = intel_fb_bo(&fb->base); struct xe_bo *bo = gem_to_xe_bo(obj); struct xe_validation_ctx ctx; struct drm_exec exec; int ret = 0; + /* We reject creating !SCANOUT fb's, so this is weird.. */ + drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC)); + if (!vma) return ERR_PTR(-ENODEV); refcount_set(&vma->ref, 1); if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && - intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && + pin_params->needs_cpu_lmem_access && !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram; @@ -376,10 +378,10 @@ static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, goto err; vma->bo = bo; - if (intel_fb_uses_dpt(&fb->base)) - ret = __xe_pin_fb_vma_dpt(fb, view, vma, alignment); + if (is_dpt) + ret = __xe_pin_fb_vma_dpt(obj, pin_params, vma); else - ret = __xe_pin_fb_vma_ggtt(fb, view, vma, alignment); + ret = __xe_pin_fb_vma_ggtt(obj, pin_params, vma); if (ret) goto err_unpin; @@ -412,97 +414,94 @@ static void __xe_unpin_fb_vma(struct i915_vma *vma) kfree(vma); } -struct i915_vma * -intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, - const struct i915_gtt_view *view, - unsigned int alignment, - unsigned int phys_alignment, - unsigned int vtd_guard, - bool uses_fence, - unsigned long *out_flags) +int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_ggtt_vma, + u32 *out_offset, + int *out_fence_id) { - *out_flags = 0; + struct i915_vma *ggtt_vma; + + ggtt_vma = __xe_pin_fb_vma(obj, false, pin_params); + if (IS_ERR(ggtt_vma)) + return PTR_ERR(ggtt_vma); + + *out_ggtt_vma = ggtt_vma; + *out_offset = xe_ggtt_node_addr(ggtt_vma->node); + if (out_fence_id) + *out_fence_id = -1; - return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, alignment); + return 0; } -void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) +static void xe_fb_pin_ggtt_unpin(struct i915_vma *ggtt_vma, + int fence_id) { - __xe_unpin_fb_vma(vma); + WARN_ON(fence_id >= 0); + + __xe_unpin_fb_vma(ggtt_vma); } -static bool reuse_vma(struct intel_plane_state *new_plane_state, - const struct intel_plane_state *old_plane_state) +static int xe_fb_pin_dpt_pin(struct drm_gem_object *obj, struct intel_dpt *dpt, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_dpt_vma, + struct i915_vma **out_ggtt_vma, + u32 *out_offset) { - struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb); - struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); - struct xe_device *xe = to_xe_device(fb->base.dev); - struct intel_display *display = xe->display; - struct i915_vma *vma; - - if (old_plane_state->hw.fb == new_plane_state->hw.fb && - !memcmp(&old_plane_state->view.gtt, - &new_plane_state->view.gtt, - sizeof(new_plane_state->view.gtt))) { - vma = old_plane_state->ggtt_vma; - goto found; - } + struct i915_vma *ggtt_vma; - if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) { - vma = intel_fbdev_vma_pointer(display->fbdev.fbdev); - if (vma) - goto found; - } + WARN_ON(dpt); - return false; + ggtt_vma = __xe_pin_fb_vma(obj, true, pin_params); + if (IS_ERR(ggtt_vma)) + return PTR_ERR(ggtt_vma); -found: - refcount_inc(&vma->ref); - new_plane_state->ggtt_vma = vma; + *out_dpt_vma = NULL; /* not used on xe */ + *out_ggtt_vma = ggtt_vma; + *out_offset = xe_ggtt_node_addr(ggtt_vma->node); - new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) + - plane->surf_offset(new_plane_state); - - return true; + return 0; } -int intel_plane_pin_fb(struct intel_plane_state *new_plane_state, - const struct intel_plane_state *old_plane_state) +static void xe_fb_pin_dpt_unpin(struct intel_dpt *dpt, + struct i915_vma *dpt_vma, + struct i915_vma *ggtt_vma) { - struct drm_framebuffer *fb = new_plane_state->hw.fb; - struct drm_gem_object *obj = intel_fb_bo(fb); - struct xe_bo *bo = gem_to_xe_bo(obj); - struct i915_vma *vma; - struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); - unsigned int alignment = plane->min_alignment(plane, fb, 0); + WARN_ON(dpt || dpt_vma); - if (reuse_vma(new_plane_state, old_plane_state)) - return 0; - - /* We reject creating !SCANOUT fb's, so this is weird.. */ - drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC)); - - vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment); - - if (IS_ERR(vma)) - return PTR_ERR(vma); + __xe_unpin_fb_vma(ggtt_vma); +} - new_plane_state->ggtt_vma = vma; +static struct i915_vma * +xe_fb_pin_reuse_vma(struct i915_vma *old_ggtt_vma, + struct drm_gem_object *old_obj, + const struct i915_gtt_view *old_view, + struct drm_gem_object *new_obj, + const struct i915_gtt_view *new_view, + u32 *out_offset) +{ + if (old_ggtt_vma && old_obj == new_obj && + !memcmp(old_view, new_view, sizeof(*new_view))) { + refcount_inc(&old_ggtt_vma->ref); - new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) + - plane->surf_offset(new_plane_state); + *out_offset = xe_ggtt_node_addr(old_ggtt_vma->node); - return 0; -} + return old_ggtt_vma; + } -void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) -{ - __xe_unpin_fb_vma(old_plane_state->ggtt_vma); - old_plane_state->ggtt_vma = NULL; + return NULL; } -void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map) +static void xe_fb_pin_get_map(struct i915_vma *vma, struct iosys_map *map) { *map = vma->bo->vmap; } + +const struct intel_display_fb_pin_interface xe_display_fb_pin_interface = { + .ggtt_pin = xe_fb_pin_ggtt_pin, + .ggtt_unpin = xe_fb_pin_ggtt_unpin, + .dpt_pin = xe_fb_pin_dpt_pin, + .dpt_unpin = xe_fb_pin_dpt_unpin, + .reuse_vma = xe_fb_pin_reuse_vma, + .get_map = xe_fb_pin_get_map, +}; diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.h b/drivers/gpu/drm/xe/display/xe_fb_pin.h new file mode 100644 index 000000000000..20dd8a99a25f --- /dev/null +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2026 Intel Corporation */ + +#ifndef __XE_FB_PIN_H__ +#define __XE_FB_PIN_H__ + +#include <linux/types.h> + +struct drm_gem_object; +struct i915_vma; +struct intel_fb_pin_params; + +int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj, + const struct intel_fb_pin_params *pin_params, + struct i915_vma **out_ggtt_vma, + u32 *out_offset, + int *out_fence_id); + +extern const struct intel_display_fb_pin_interface xe_display_fb_pin_interface; + +#endif /* __XE_FB_PIN_H__ */ diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c index 29c72aa4b0d2..33494b86205d 100644 --- a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c +++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c @@ -37,9 +37,17 @@ static bool intel_hdcp_gsc_check_status(struct drm_device *drm) struct xe_device *xe = to_xe_device(drm); struct xe_tile *tile = xe_device_get_root_tile(xe); struct xe_gt *gt = tile->media_gt; - struct xe_gsc *gsc = >->uc.gsc; + struct xe_gsc *gsc; + + if (!gt) { + drm_dbg_kms(&xe->drm, + "not checking GSC status for HDCP2.x: media GT not present or disabled\n"); + return false; + } + + gsc = >->uc.gsc; - if (!gsc || !xe_uc_fw_is_available(&gsc->fw)) { + if (!xe_uc_fw_is_available(&gsc->fw)) { drm_dbg_kms(&xe->drm, "GSC Components not ready for HDCP2.x\n"); return false; diff --git a/drivers/gpu/drm/xe/display/xe_initial_plane.c b/drivers/gpu/drm/xe/display/xe_initial_plane.c index 8bcae552dddc..02b46cc3b6df 100644 --- a/drivers/gpu/drm/xe/display/xe_initial_plane.c +++ b/drivers/gpu/drm/xe/display/xe_initial_plane.c @@ -7,34 +7,20 @@ #include "regs/xe_gtt_defs.h" -#include "intel_crtc.h" -#include "intel_display_regs.h" -#include "intel_display_types.h" +/* FIXME move intel_remapped_info_size() & co. */ #include "intel_fb.h" -#include "intel_fb_pin.h" -#include "intel_fbdev_fb.h" + +/* FIXME move intel_initial_plane_config */ +#include "intel_display_types.h" + #include "xe_bo.h" +#include "xe_display_bo.h" #include "xe_display_vma.h" +#include "xe_fb_pin.h" #include "xe_ggtt.h" #include "xe_mmio.h" #include "xe_vram_types.h" -/* Early xe has no irq */ -static void xe_initial_plane_vblank_wait(struct drm_crtc *_crtc) -{ - struct intel_crtc *crtc = to_intel_crtc(_crtc); - struct xe_device *xe = to_xe_device(crtc->base.dev); - struct xe_reg pipe_frmtmstmp = XE_REG(i915_mmio_reg_offset(PIPE_FRMTMSTMP(crtc->pipe))); - u32 timestamp; - int ret; - - timestamp = xe_mmio_read32(xe_root_tile_mmio(xe), pipe_frmtmstmp); - - ret = xe_mmio_wait32_not(xe_root_tile_mmio(xe), pipe_frmtmstmp, ~0U, timestamp, 40000U, ×tamp, false); - if (ret < 0) - drm_warn(&xe->drm, "waiting for early vblank failed with %i\n", ret); -} - static struct xe_bo * initial_plane_bo(struct xe_device *xe, struct intel_initial_plane_config *plane_config) @@ -86,7 +72,8 @@ initial_plane_bo(struct xe_device *xe, flags |= XE_BO_FLAG_STOLEN; if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) && - !intel_fbdev_fb_prefer_stolen(&xe->drm, plane_config->size)) { + IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) && + !xe_display_bo_fbdev_prefer_stolen(xe, plane_config->size)) { drm_info(&xe->drm, "Initial FB size exceeds half of stolen, discarding\n"); return NULL; } @@ -114,7 +101,7 @@ xe_alloc_initial_plane_obj(struct drm_device *drm, { struct xe_device *xe = to_xe_device(drm); struct drm_mode_fb_cmd2 mode_cmd = { 0 }; - struct drm_framebuffer *fb = &plane_config->fb->base; + struct drm_framebuffer *fb = plane_config->fb; struct xe_bo *bo; mode_cmd.pixel_format = fb->format->format; @@ -151,15 +138,19 @@ xe_initial_plane_setup(struct drm_plane_state *_plane_state, { struct intel_plane_state *plane_state = to_intel_plane_state(_plane_state); struct i915_vma *vma; + struct intel_fb_pin_params pin_params = { + .view = &plane_state->view.gtt, + }; + u32 offset; + int ret; - vma = intel_fb_pin_to_ggtt(fb, &plane_state->view.gtt, - 0, 0, 0, false, &plane_state->flags); - if (IS_ERR(vma)) - return PTR_ERR(vma); + ret = xe_fb_pin_ggtt_pin(intel_fb_bo(fb), &pin_params, &vma, &offset, NULL); + if (ret) + return ret; plane_state->ggtt_vma = vma; - plane_state->surf = xe_ggtt_node_addr(plane_state->ggtt_vma->node); + plane_state->surf = offset; plane_config->vma = vma; @@ -171,7 +162,6 @@ static void xe_plane_config_fini(struct intel_initial_plane_config *plane_config } const struct intel_display_initial_plane_interface xe_display_initial_plane_interface = { - .vblank_wait = xe_initial_plane_vblank_wait, .alloc_obj = xe_alloc_initial_plane_obj, .setup = xe_initial_plane_setup, .config_fini = xe_plane_config_fini, diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index 885fcf211e6d..18d0fde8c98f 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -20,7 +20,6 @@ #define XY_FAST_COLOR_BLT_CMD (2 << 29 | 0x44 << 22) #define XY_FAST_COLOR_BLT_DEPTH_32 (2 << 19) -#define XY_FAST_COLOR_BLT_DW 16 #define XY_FAST_COLOR_BLT_MOCS_MASK GENMASK(27, 22) #define XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK GENMASK(27, 24) #define XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT 31 @@ -31,14 +30,13 @@ #define XY_FAST_COPY_BLT_D1_DST_TILE4 REG_BIT(30) #define XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK GENMASK(23, 20) -#define MEM_COPY_CMD (2 << 29 | 0x5a << 22 | 0x8) +#define MEM_COPY_CMD (2 << 29 | 0x5a << 22) #define MEM_COPY_PAGE_COPY_MODE REG_BIT(19) #define MEM_COPY_MATRIX_COPY REG_BIT(17) #define MEM_COPY_SRC_MOCS_INDEX_MASK GENMASK(31, 28) #define MEM_COPY_DST_MOCS_INDEX_MASK GENMASK(6, 3) #define PVC_MEM_SET_CMD (2 << 29 | 0x5b << 22) -#define PVC_MEM_SET_CMD_LEN_DW 7 #define PVC_MEM_SET_MATRIX REG_BIT(17) #define PVC_MEM_SET_DATA_FIELD GENMASK(31, 24) /* Bspec lists field as [6:0], but index alone is from [6:1] */ diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h index 1b4a7e9a703d..94033982e694 100644 --- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -165,18 +165,21 @@ #define CTX_CTRL_INHIBIT_SYN_CTX_SWITCH REG_BIT(3) #define CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT REG_BIT(0) -#define RING_MODE(base) XE_REG((base) + 0x29c) -#define GFX_DISABLE_LEGACY_MODE REG_BIT(3) +#define GFX_MODE(base) XE_REG((base) + 0x29c, XE_REG_OPTION_MASKED) #define GFX_MSIX_INTERRUPT_ENABLE REG_BIT(13) +#define GFX_DISABLE_LEGACY_MODE REG_BIT(3) #define RING_CSMQDEBUG(base) XE_REG((base) + 0x2b0) +#define CURRENT_ACTIVE_QUEUE_ID_MASK REG_GENMASK(7, 0) + +#define RING_QUEUE_TIMESTAMP(base) XE_REG((base) + 0x4c0) +#define RING_QUEUE_TIMESTAMP_UDW(base) XE_REG((base) + 0x4c0 + 4) #define RING_TIMESTAMP(base) XE_REG((base) + 0x358) #define RING_TIMESTAMP_UDW(base) XE_REG((base) + 0x358 + 4) #define RING_VALID_MASK 0x00000001 #define RING_VALID 0x00000001 -#define STOP_RING REG_BIT(8) #define RING_CTX_TIMESTAMP(base) XE_REG((base) + 0x3a8) #define RING_CTX_TIMESTAMP_UDW(base) XE_REG((base) + 0x3ac) diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 4ebaa0888a43..08251c7a1a4b 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -152,10 +152,11 @@ #define XEHPG_INSTDONE_GEOM_SVGUNIT XE_REG_MCR(0x666c) -#define CACHE_MODE_1 XE_REG(0x7004, XE_REG_OPTION_MASKED) +#define CACHE_MODE_1 XE_REG_MCR(0x7004, XE_REG_OPTION_MASKED) #define MSAA_OPTIMIZATION_REDUC_DISABLE REG_BIT(11) #define COMMON_SLICE_CHICKEN1 XE_REG(0x7010, XE_REG_OPTION_MASKED) +#define XEHP_COMMON_SLICE_CHICKEN1 XE_REG_MCR(0x7010, XE_REG_OPTION_MASKED) #define DISABLE_BOTTOM_CLIP_RECTANGLE_TEST REG_BIT(14) #define HIZ_CHICKEN XE_REG(0x7018, XE_REG_OPTION_MASKED) @@ -178,6 +179,7 @@ #define XEHPG_SC_INSTDONE_EXTRA2 XE_REG_MCR(0x7108) #define COMMON_SLICE_CHICKEN4 XE_REG(0x7300, XE_REG_OPTION_MASKED) +#define XEHP_COMMON_SLICE_CHICKEN4 XE_REG_MCR(0x7300, XE_REG_OPTION_MASKED) #define SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE REG_BIT(12) #define DISABLE_TDC_LOAD_BALANCING_CALC REG_BIT(6) #define HW_FILTERING REG_BIT(5) @@ -528,6 +530,7 @@ #define ROW_CHICKEN3 XE_REG_MCR(0xe49c, XE_REG_OPTION_MASKED) #define XE2_EUPEND_CHK_FLUSH_DIS REG_BIT(14) +#define DIS_EU_GRF_POISON_TO_LSC REG_BIT(13) #define DIS_FIX_EOT1_FLUSH REG_BIT(9) #define TDL_TSL_CHICKEN XE_REG_MCR(0xe4c4, XE_REG_OPTION_MASKED) @@ -535,6 +538,9 @@ #define SLM_WMTP_RESTORE REG_BIT(11) #define RES_CHK_SPR_DIS REG_BIT(6) +#define TDL_TSL_CHICKEN2 XE_REG_MCR(0xe4cc, XE_REG_OPTION_MASKED) +#define TILEY_LOCALID REG_BIT(2) + #define ROW_CHICKEN XE_REG_MCR(0xe4f0, XE_REG_OPTION_MASKED) #define UGM_BACKUP_MODE REG_BIT(13) #define MDQ_ARBITRATION_MODE REG_BIT(12) @@ -560,6 +566,7 @@ #define DIS_ATOMIC_CHAINING_TYPED_WRITES REG_BIT(3) #define TDL_CHICKEN XE_REG_MCR(0xe5f4, XE_REG_OPTION_MASKED) +#define BIT_APQ_OPT_DIS REG_BIT(14) #define QID_WAIT_FOR_THREAD_NOT_RUN_DISABLE REG_BIT(12) #define EUSTALL_PERF_SAMPLING_DISABLE REG_BIT(5) @@ -583,7 +590,7 @@ #define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32) #define LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE REG_BIT(35 - 32) -#define ROW_CHICKEN5 XE_REG_MCR(0xe7f0) +#define ROW_CHICKEN5 XE_REG_MCR(0xe7f0, XE_REG_OPTION_MASKED) #define CPSS_AWARE_DIS REG_BIT(3) #define SARB_CHICKEN1 XE_REG_MCR(0xe90c) diff --git a/drivers/gpu/drm/xe/regs/xe_irq_regs.h b/drivers/gpu/drm/xe/regs/xe_irq_regs.h index 9d74f454d3ff..1d6b976c4de0 100644 --- a/drivers/gpu/drm/xe/regs/xe_irq_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_irq_regs.h @@ -22,6 +22,7 @@ #define DISPLAY_IRQ REG_BIT(16) #define SOC_H2DMEMINT_IRQ REG_BIT(13) #define I2C_IRQ REG_BIT(12) +#define SYSCTRL_IRQ REG_BIT(11) #define GT_DW_IRQ(x) REG_BIT(x) /* diff --git a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h index b5eff383902c..4ab86fc369fd 100644 --- a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h +++ b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h @@ -34,6 +34,9 @@ #define CTX_CS_INT_VEC_REG 0x5a #define CTX_CS_INT_VEC_DATA (CTX_CS_INT_VEC_REG + 1) +#define CTX_QUEUE_TIMESTAMP (0xd0 + 1) +#define CTX_QUEUE_TIMESTAMP_UDW (0xd2 + 1) + #define INDIRECT_CTX_RING_HEAD (0x02 + 1) #define INDIRECT_CTX_RING_TAIL (0x04 + 1) #define INDIRECT_CTX_RING_START (0x06 + 1) diff --git a/drivers/gpu/drm/xe/regs/xe_oa_regs.h b/drivers/gpu/drm/xe/regs/xe_oa_regs.h index 04a729e610aa..aa66af7e99fe 100644 --- a/drivers/gpu/drm/xe/regs/xe_oa_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_oa_regs.h @@ -6,6 +6,9 @@ #ifndef __XE_OA_REGS__ #define __XE_OA_REGS__ +#define SYS_MEM_LAT_MEASURE XE_REG(0x145194) +#define SYS_MEM_LAT_MEASURE_EN REG_BIT(31) + #define RPM_CONFIG1 XE_REG(0xd04) #define GT_NOA_ENABLE REG_BIT(9) diff --git a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h index 4b3c46eb858f..c63b409d7a82 100644 --- a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h @@ -27,4 +27,7 @@ #define TEMP_SIGN_MASK REG_BIT(31) #define BMG_PACKAGE_TEMPERATURE XE_REG(0x138434) +#define CRI_PACKAGE_ENERGY_STATUS XE_REG(0x138120) +#define CRI_PLATFORM_ENERGY_STATUS XE_REG(0x138458) + #endif /* _XE_PCODE_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h b/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h new file mode 100644 index 000000000000..59f3f3ec59a6 --- /dev/null +++ b/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_REGS_H_ +#define _XE_SYSCTRL_REGS_H_ + +#include "xe_regs.h" + +#define SYSCTRL_BASE_OFFSET 0xdb000 +#define SYSCTRL_BASE (SOC_BASE + SYSCTRL_BASE_OFFSET) +#define SYSCTRL_MAILBOX_INDEX 0x03 +#define SYSCTRL_BAR_LENGTH 0x1000 + +#define SYSCTRL_MB_CTRL XE_REG(0x10) +#define SYSCTRL_MB_CTRL_RUN_BUSY REG_BIT(31) +#define SYSCTRL_MB_CTRL_IRQ REG_BIT(30) +#define SYSCTRL_MB_CTRL_RUN_BUSY_OUT REG_BIT(29) +#define SYSCTRL_MB_CTRL_PARAM3_MASK REG_GENMASK(28, 24) +#define SYSCTRL_MB_CTRL_PARAM2_MASK REG_GENMASK(23, 16) +#define SYSCTRL_MB_CTRL_PARAM1_MASK REG_GENMASK(15, 8) +#define SYSCTRL_MB_CTRL_COMMAND_MASK REG_GENMASK(7, 0) +#define SYSCTRL_MB_CTRL_CMD REG_FIELD_PREP(SYSCTRL_MB_CTRL_COMMAND_MASK, 5) + +#define SYSCTRL_MB_DATA0 XE_REG(0x14) +#define SYSCTRL_MB_DATA1 XE_REG(0x18) +#define SYSCTRL_MB_DATA2 XE_REG(0x1c) +#define SYSCTRL_MB_DATA3 XE_REG(0x20) + +#define SYSCTRL_FRAME_PHASE REG_BIT(24) +#define SYSCTRL_FRAME_CURRENT_MASK REG_GENMASK(21, 16) +#define SYSCTRL_FRAME_TOTAL_MASK REG_GENMASK(13, 8) +#define SYSCTRL_FRAME_COMMAND_MASK REG_GENMASK(7, 0) + +#endif diff --git a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c index efa8963ec248..e6eaa94d4d30 100644 --- a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c +++ b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c @@ -13,11 +13,28 @@ #define TEST_MAX_VFS 63 #define TEST_VRAM 0x7a800000ull /* random size that works on 32-bit */ +static bool xe_device_is_admin_only_stub_enable(const struct xe_device *xe) +{ + return true; +} + +static bool xe_device_is_admin_only_stub_disable(const struct xe_device *xe) +{ + return false; +} + static void pf_set_admin_mode(struct xe_device *xe, bool enable) { - /* should match logic of xe_sriov_pf_admin_only() */ - xe->sriov.pf.admin_only = enable; + typeof(xe_device_is_admin_only) *stub = enable ? + xe_device_is_admin_only_stub_enable : + xe_device_is_admin_only_stub_disable; + + kunit_activate_static_stub(kunit_get_current_test(), + xe_device_is_admin_only, + *stub); + KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_sriov_pf_admin_only(xe)); + KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_device_is_admin_only(xe)); } static void pf_set_usable_vram(struct xe_device *xe, u64 usable) diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 34e2f0f4631f..3c1be809be82 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -9,6 +9,7 @@ #include "tests/xe_kunit_helpers.h" #include "tests/xe_pci_test.h" +#include "xe_pat.h" #include "xe_pci.h" #include "xe_pm.h" @@ -246,7 +247,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test, /* First part of the test, are we updating our pagetable bo with a new entry? */ xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64, 0xdeaddeadbeefbeef); - expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe->pat.idx[XE_CACHE_WB], 0); + expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe_cache_pat_idx(xe, XE_CACHE_WB), 0); if (m->q->vm->flags & XE_VM_FLAG_64K) expected |= XE_PTE_PS64; if (xe_bo_is_vram(pt)) @@ -420,7 +421,7 @@ static struct dma_fence *blt_copy(struct xe_tile *tile, avail_pts, avail_pts); /* Add copy commands size here */ - batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + + batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) + ((xe_device_has_flat_ccs(xe) && copy_only_ccs) ? EMIT_COPY_CCS_DW : 0); bb = xe_bb_new(gt, batch_size, xe->info.has_usm); diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c index f3179b31f13e..860409c579f8 100644 --- a/drivers/gpu/drm/xe/tests/xe_pci.c +++ b/drivers/gpu/drm/xe/tests/xe_pci.c @@ -131,12 +131,12 @@ static const char *subplatform_prefix(enum xe_subplatform s) return s == XE_SUBPLATFORM_NONE ? "" : " "; } -static const char *step_prefix(enum xe_step step) +static const char *step_prefix(enum intel_step step) { return step == STEP_NONE ? "" : " "; } -static const char *step_name(enum xe_step step) +static const char *step_name(enum intel_step step) { return step == STEP_NONE ? "" : xe_step_name(step); } diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_test.c b/drivers/gpu/drm/xe/tests/xe_rtp_test.c index e5a0f985a700..5d78f2283df9 100644 --- a/drivers/gpu/drm/xe/tests/xe_rtp_test.c +++ b/drivers/gpu/drm/xe/tests/xe_rtp_test.c @@ -9,24 +9,30 @@ #include <drm/drm_drv.h> #include <drm/drm_kunit_helpers.h> +#include <kunit/static_stub.h> #include <kunit/test.h> #include "regs/xe_gt_regs.h" #include "regs/xe_reg_defs.h" #include "xe_device.h" #include "xe_device_types.h" +#include "xe_gt_mcr.h" #include "xe_kunit_helpers.h" #include "xe_pci_test.h" #include "xe_reg_sr.h" #include "xe_rtp.h" -#define REGULAR_REG1 XE_REG(1) -#define REGULAR_REG2 XE_REG(2) -#define REGULAR_REG3 XE_REG(3) -#define MCR_REG1 XE_REG_MCR(1) -#define MCR_REG2 XE_REG_MCR(2) -#define MCR_REG3 XE_REG_MCR(3) -#define MASKED_REG1 XE_REG(1, XE_REG_OPTION_MASKED) +#define REGULAR_REG1 XE_REG(1) +#define REGULAR_REG2 XE_REG(2) +#define REGULAR_REG3 XE_REG(3) +#define REGULAR_REG4 XE_REG(4) +#define BAD_REGULAR_REG5 XE_REG(5) +#define MCR_REG1 XE_REG_MCR(1) +#define MCR_REG2 XE_REG_MCR(2) +#define MCR_REG3 XE_REG_MCR(3) +#define BAD_MCR_REG4 XE_REG_MCR(4) +#define MCR_REG5 XE_REG_MCR(5) +#define MASKED_REG1 XE_REG(1, XE_REG_OPTION_MASKED) #undef XE_REG_MCR #define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1) @@ -48,6 +54,23 @@ struct rtp_test_case { const struct xe_rtp_entry *entries; }; +static bool fake_xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg) +{ + /* + * All supported platforms in this imaginary setup will always have REG4 + * as a non-MCR register and REG5 as MCR, meaning that BAD_MCR_REG4 and + * BAD_REGULAR_REG5 represent programming errors to be captured by our + * tests. + */ + if (reg.raw == BAD_REGULAR_REG5.raw) + return true; + + if (reg.raw == BAD_MCR_REG4.raw) + return false; + + return reg.mcr; +} + static bool match_yes(const struct xe_device *xe, const struct xe_gt *gt, const struct xe_hw_engine *hwe) { @@ -304,6 +327,38 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = { {} }, }, + { + .name = "bad-mcr-reg-forced-to-regular", + .expected_reg = REGULAR_REG4, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .entries = (const struct xe_rtp_entry_sr[]) { + { XE_RTP_NAME("bad-mcr-regular-reg"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(BAD_MCR_REG4, REG_BIT(0))) + }, + {} + }, + }, + { + .name = "bad-regular-reg-forced-to-mcr", + .expected_reg = MCR_REG5, + .expected_set_bits = REG_BIT(0), + .expected_clr_bits = REG_BIT(0), + .expected_active = BIT(0), + .expected_count_sr_entries = 1, + .expected_sr_errors = 1, + .entries = (const struct xe_rtp_entry_sr[]) { + { XE_RTP_NAME("bad-regular-reg"), + XE_RTP_RULES(FUNC(match_yes)), + XE_RTP_ACTIONS(SET(BAD_REGULAR_REG5, REG_BIT(0))) + }, + {} + }, + }, }; static void xe_rtp_process_to_sr_tests(struct kunit *test) @@ -523,6 +578,8 @@ static int xe_rtp_test_init(struct kunit *test) xe->drm.dev = dev; test->priv = xe; + kunit_activate_static_stub(test, xe_gt_mcr_check_reg, fake_xe_gt_mcr_check_reg); + return 0; } diff --git a/drivers/gpu/drm/xe/tests/xe_wa_test.c b/drivers/gpu/drm/xe/tests/xe_wa_test.c index 49d191043dfa..ff0e2502b39f 100644 --- a/drivers/gpu/drm/xe/tests/xe_wa_test.c +++ b/drivers/gpu/drm/xe/tests/xe_wa_test.c @@ -9,6 +9,8 @@ #include <kunit/test.h> #include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" #include "xe_kunit_helpers.h" #include "xe_pci_test.h" #include "xe_reg_sr.h" @@ -19,8 +21,10 @@ static int xe_wa_test_init(struct kunit *test) { const struct xe_pci_fake_data *param = test->param_value; struct xe_pci_fake_data data = *param; - struct xe_device *xe; struct device *dev; + struct xe_device *xe; + struct xe_gt *gt; + int id; int ret; dev = drm_kunit_helper_alloc_device(test); @@ -33,6 +37,12 @@ static int xe_wa_test_init(struct kunit *test) ret = xe_pci_fake_device_init(xe); KUNIT_ASSERT_EQ(test, ret, 0); + /* Needed for sanitize_mcr(). */ + for_each_gt(gt, xe, id) { + xe_gt_mcr_init_early(gt); + xe_gt_mmio_init(gt); + } + if (!param->graphics_verx100) xe->info.step = param->step; @@ -55,7 +65,7 @@ static void xe_wa_gt(struct kunit *test) xe_wa_process_gt(gt); xe_tuning_process_gt(gt); - KUNIT_ASSERT_EQ(test, gt->reg_sr.errors, 0); + KUNIT_EXPECT_EQ(test, gt->reg_sr.errors, 0); } } diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index a7c2dc7f224c..4c80bac67622 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -173,19 +173,6 @@ mem_type_to_migrate(struct xe_device *xe, u32 mem_type) return tile->migrate; } -static struct xe_vram_region *res_to_mem_region(struct ttm_resource *res) -{ - struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); - struct ttm_resource_manager *mgr; - struct xe_ttm_vram_mgr *vram_mgr; - - xe_assert(xe, resource_is_vram(res)); - mgr = ttm_manager_type(&xe->ttm, res->mem_type); - vram_mgr = to_xe_ttm_vram_mgr(mgr); - - return container_of(vram_mgr, struct xe_vram_region, ttm); -} - static void try_add_system(struct xe_device *xe, struct xe_bo *bo, u32 bo_flags, u32 *c) { @@ -599,11 +586,17 @@ static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt) kfree(tt); } -static bool xe_ttm_resource_visible(struct ttm_resource *mem) +static bool xe_ttm_resource_visible(struct xe_device *xe, struct ttm_resource *mem) { - struct xe_ttm_vram_mgr_resource *vres = - to_xe_ttm_vram_mgr_resource(mem); + struct xe_ttm_vram_mgr_resource *vres; + + if (mem->mem_type == XE_PL_STOLEN) { + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; + return mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe); + } + + vres = to_xe_ttm_vram_mgr_resource(mem); return vres->used_visible_size == mem->size; } @@ -621,7 +614,7 @@ bool xe_bo_is_visible_vram(struct xe_bo *bo) if (drm_WARN_ON(bo->ttm.base.dev, !xe_bo_is_vram(bo))) return false; - return xe_ttm_resource_visible(bo->ttm.resource); + return xe_ttm_resource_visible(xe_bo_device(bo), bo->ttm.resource); } static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, @@ -635,9 +628,9 @@ static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, return 0; case XE_PL_VRAM0: case XE_PL_VRAM1: { - struct xe_vram_region *vram = res_to_mem_region(mem); + struct xe_vram_region *vram = xe_map_resource_to_region(mem); - if (!xe_ttm_resource_visible(mem)) + if (!xe_ttm_resource_visible(xe, mem)) return -EINVAL; mem->bus.offset = mem->start << PAGE_SHIFT; @@ -897,10 +890,10 @@ void xe_bo_set_purgeable_state(struct xe_bo *bo, new_state == XE_MADV_PURGEABLE_PURGED); /* Once purged, always purged - cannot transition out */ - xe_assert(xe, !(bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED && + xe_assert(xe, !(bo->purgeable.state == XE_MADV_PURGEABLE_PURGED && new_state != XE_MADV_PURGEABLE_PURGED)); - bo->madv_purgeable = new_state; + bo->purgeable.state = new_state; xe_bo_set_purgeable_shrinker(bo, new_state); } @@ -1642,7 +1635,7 @@ static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo, if (ttm_bo->resource->mem_type == XE_PL_STOLEN) return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT; - vram = res_to_mem_region(ttm_bo->resource); + vram = xe_map_resource_to_region(ttm_bo->resource); xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor); return (vram->io_start + cursor.start) >> PAGE_SHIFT; } @@ -1782,7 +1775,7 @@ static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo, goto out; } - vram = res_to_mem_region(ttm_bo->resource); + vram = xe_map_resource_to_region(ttm_bo->resource); xe_res_first(ttm_bo->resource, offset & PAGE_MASK, xe_bo_size(bo) - (offset & PAGE_MASK), &cursor); @@ -2322,8 +2315,10 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, } /* XE_BO_FLAG_GGTTx requires XE_BO_FLAG_GGTT also be set */ - if ((flags & XE_BO_FLAG_GGTT_ALL) && !(flags & XE_BO_FLAG_GGTT)) + if ((flags & XE_BO_FLAG_GGTT_ALL) && !(flags & XE_BO_FLAG_GGTT)) { + xe_bo_free(bo); return ERR_PTR(-EINVAL); + } if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) && !(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) && @@ -2342,8 +2337,10 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, alignment = SZ_4K >> PAGE_SHIFT; } - if (type == ttm_bo_type_device && aligned_size != size) + if (type == ttm_bo_type_device && aligned_size != size) { + xe_bo_free(bo); return ERR_PTR(-EINVAL); + } if (!bo) { bo = xe_bo_alloc(); @@ -2364,7 +2361,7 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, INIT_LIST_HEAD(&bo->vram_userfault_link); /* Initialize purge advisory state */ - bo->madv_purgeable = XE_MADV_PURGEABLE_WILLNEED; + bo->purgeable.state = XE_MADV_PURGEABLE_WILLNEED; drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size); @@ -2923,7 +2920,7 @@ uint64_t vram_region_gpu_offset(struct ttm_resource *res) case XE_PL_SYSTEM: return 0; default: - return res_to_mem_region(res)->dpa_base; + return xe_map_resource_to_region(res)->dpa_base; } return 0; } diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 68dea7d25a6b..6340317f7d2e 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -251,7 +251,7 @@ static inline bool xe_bo_is_protected(const struct xe_bo *bo) static inline bool xe_bo_is_purged(struct xe_bo *bo) { xe_bo_assert_held(bo); - return bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED; + return bo->purgeable.state == XE_MADV_PURGEABLE_PURGED; } /** @@ -268,11 +268,95 @@ static inline bool xe_bo_is_purged(struct xe_bo *bo) static inline bool xe_bo_madv_is_dontneed(struct xe_bo *bo) { xe_bo_assert_held(bo); - return bo->madv_purgeable == XE_MADV_PURGEABLE_DONTNEED; + return bo->purgeable.state == XE_MADV_PURGEABLE_DONTNEED; } void xe_bo_set_purgeable_state(struct xe_bo *bo, enum xe_madv_purgeable_state new_state); +/** + * xe_bo_willneed_get_locked() - Acquire a WILLNEED holder on a BO + * @bo: Buffer object + * + * Increments willneed_count and, on a 0->1 transition, promotes the BO + * from DONTNEED to WILLNEED. PURGED is terminal and is never modified. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_willneed_get_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + /* Imported BOs are owned externally; do not track purgeability. */ + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + if (bo->purgeable.willneed_count++ == 0 && xe_bo_madv_is_dontneed(bo)) + xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_WILLNEED); +} + +/** + * xe_bo_willneed_put_locked() - Release a WILLNEED holder on a BO + * @bo: Buffer object + * + * Decrements willneed_count and, on a 1->0 transition, marks the BO + * DONTNEED only if it still has VMAs (implying all active VMAs are + * DONTNEED). If the last VMA is being removed, preserve the current BO + * state to match the previous VMA-walk semantics. + * + * PURGED is terminal and the BO state is never modified. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_willneed_put_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + xe_assert(xe_bo_device(bo), bo->purgeable.willneed_count > 0); + if (--bo->purgeable.willneed_count == 0 && bo->purgeable.vma_count > 0 && + !xe_bo_is_purged(bo)) + xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_DONTNEED); +} + +/** + * xe_bo_vma_count_inc_locked() - Account a new VMA on a BO + * @bo: Buffer object + * + * Increments vma_count. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_vma_count_inc_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + bo->purgeable.vma_count++; +} + +/** + * xe_bo_vma_count_dec_locked() - Account a VMA removal on a BO + * @bo: Buffer object + * + * Decrements vma_count. + * + * Caller must hold the BO's dma-resv lock. + */ +static inline void xe_bo_vma_count_dec_locked(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + + if (drm_gem_is_imported(&bo->ttm.base)) + return; + + xe_assert(xe_bo_device(bo), bo->purgeable.vma_count > 0); + bo->purgeable.vma_count--; +} + static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo) { if (likely(bo)) { diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index ff8317bfc1ae..fcc63ae3f455 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -18,6 +18,7 @@ #include "xe_ggtt_types.h" struct xe_device; +struct xe_mem_pool_node; struct xe_vm; #define XE_BO_MAX_PLACEMENTS 3 @@ -66,7 +67,7 @@ struct xe_bo { /** @attr: User controlled attributes for bo */ struct { /** - * @atomic_access: type of atomic access bo needs + * @attr.atomic_access: type of atomic access bo needs * protected by bo dma-resv lock */ u32 atomic_access; @@ -88,7 +89,7 @@ struct xe_bo { bool ccs_cleared; /** @bb_ccs: BB instructions of CCS read/write. Valid only for VF */ - struct xe_bb *bb_ccs[XE_SRIOV_VF_CCS_CTX_COUNT]; + struct xe_mem_pool_node *bb_ccs[XE_SRIOV_VF_CCS_CTX_COUNT]; /** * @cpu_caching: CPU caching mode. Currently only used for userspace @@ -110,10 +111,32 @@ struct xe_bo { u64 min_align; /** - * @madv_purgeable: user space advise on BO purgeability, protected - * by BO's dma-resv lock. + * @purgeable: Purgeability state and accounting. + * + * All fields are protected by the BO's dma-resv lock. */ - u32 madv_purgeable; + struct { + /** + * @purgeable.state: BO purgeability state + * (WILLNEED/DONTNEED/PURGED). + */ + u32 state; + + /** + * @purgeable.vma_count: Number of VMAs currently mapping this BO. + */ + u32 vma_count; + + /** + * @purgeable.willneed_count: Number of active WILLNEED holders. + * + * Counts WILLNEED VMAs plus active dma-buf exports for + * non-imported BOs. The BO flips to DONTNEED on a 1->0 + * transition only when VMAs still exist; if the last VMA is + * removed, the previous BO state is preserved. + */ + u32 willneed_count; + } purgeable; }; #endif diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c index ad2d8f179eb6..22b471303984 100644 --- a/drivers/gpu/drm/xe/xe_debugfs.c +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -15,9 +15,11 @@ #include "xe_bo.h" #include "xe_device.h" #include "xe_force_wake.h" +#include "xe_gt.h" #include "xe_gt_debugfs.h" #include "xe_gt_printk.h" #include "xe_guc_ads.h" +#include "xe_hw_engine.h" #include "xe_mmio.h" #include "xe_pm.h" #include "xe_psmi.h" @@ -61,6 +63,37 @@ static struct xe_device *node_to_xe(struct drm_info_node *node) return to_xe_device(node->minor->dev); } +static void print_engine_class_mask(struct drm_printer *p, u16 mask) +{ + if (!mask) { + drm_printf(p, " none\n"); + return; + } + + for (enum xe_engine_class ec = 0; ec < XE_ENGINE_CLASS_MAX; ec++) { + if (mask & BIT(ec)) + drm_printf(p, " %s", xe_hw_engine_class_to_str(ec)); + } + drm_printf(p, "\n"); +} + +static void print_engine_mask(struct drm_printer *p, struct xe_gt *gt, u64 mask) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + if (!mask) { + drm_printf(p, " none\n"); + return; + } + + for_each_hw_engine(hwe, gt, id) { + if (mask & BIT_ULL(id)) + drm_printf(p, " %s", hwe->name); + } + drm_printf(p, "\n"); +} + static int info(struct seq_file *m, void *data) { struct xe_device *xe = node_to_xe(m->private); @@ -88,13 +121,15 @@ static int info(struct seq_file *m, void *data) drm_printf(&p, "has_flat_ccs %s\n", str_yes_no(xe->info.has_flat_ccs)); drm_printf(&p, "has_usm %s\n", str_yes_no(xe->info.has_usm)); drm_printf(&p, "skip_guc_pc %s\n", str_yes_no(xe->info.skip_guc_pc)); + drm_printf(&p, "multi_lrc_engine_classes"); + print_engine_class_mask(&p, xe->info.multi_lrc_mask); for_each_gt(gt, xe, id) { drm_printf(&p, "gt%d force wake %d\n", id, xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT)); - drm_printf(&p, "gt%d engine_mask 0x%llx\n", id, - gt->info.engine_mask); - drm_printf(&p, "gt%d multi_queue_engine_class_mask 0x%x\n", id, - gt->info.multi_queue_engine_class_mask); + drm_printf(&p, "gt%d engines", id); + print_engine_mask(&p, gt, gt->info.engine_mask); + drm_printf(&p, "gt%d multi_queue_engine_classes", id); + print_engine_class_mask(&p, gt->info.multi_queue_engine_class_mask); } return 0; diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.c b/drivers/gpu/drm/xe/xe_dep_scheduler.c index 51d99fee9aa5..004aac8b89e6 100644 --- a/drivers/gpu/drm/xe/xe_dep_scheduler.c +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.c @@ -78,7 +78,6 @@ xe_dep_scheduler_create(struct xe_device *xe, const struct drm_sched_init_args args = { .ops = &sched_ops, .submit_wq = submit_wq, - .num_rqs = 1, .credit_limit = job_limit, .timeout = MAX_SCHEDULE_TIMEOUT, .name = name, diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.h b/drivers/gpu/drm/xe/xe_dep_scheduler.h index 853961eec64b..f314fb5d80f5 100644 --- a/drivers/gpu/drm/xe/xe_dep_scheduler.h +++ b/drivers/gpu/drm/xe/xe_dep_scheduler.h @@ -3,6 +3,9 @@ * Copyright © 2025 Intel Corporation */ +#ifndef _XE_DEP_SCHEDULER_H_ +#define _XE_DEP_SCHEDULER_H_ + #include <linux/types.h> struct drm_sched_entity; @@ -19,3 +22,5 @@ void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler); struct drm_sched_entity * xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler); + +#endif diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c index 558a1a9841a0..5f2b90b18f97 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.c +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -8,7 +8,7 @@ #include <linux/ascii85.h> #include <linux/devcoredump.h> -#include <generated/utsrelease.h> +#include <linux/utsname.h> #include <drm/drm_managed.h> @@ -101,7 +101,7 @@ static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count, drm_puts(&p, "**** Xe Device Coredump ****\n"); drm_printf(&p, "Reason: %s\n", ss->reason); - drm_puts(&p, "kernel: " UTS_RELEASE "\n"); + drm_printf(&p, "kernel: %s\n", init_utsname()->release); drm_puts(&p, "module: " KBUILD_MODNAME "\n"); ts = ktime_to_timespec64(ss->snapshot_time); @@ -149,7 +149,8 @@ static void xe_devcoredump_snapshot_free(struct xe_devcoredump_snapshot *ss) xe_guc_ct_snapshot_free(ss->guc.ct); ss->guc.ct = NULL; - xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc); + if (!IS_ERR_OR_NULL(ss->gt)) + xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc); ss->matched_node = NULL; xe_guc_exec_queue_snapshot_free(ss->ge); @@ -254,7 +255,8 @@ static void xe_devcoredump_free(void *data) if (!data || !coredump_to_xe(coredump)) return; - cancel_work_sync(&coredump->snapshot.work); + if (coredump->captured) + cancel_work_sync(&coredump->snapshot.work); mutex_lock(&coredump->lock); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index ffea4a453c01..576095cf0952 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -10,13 +10,13 @@ #include <linux/fault-inject.h> #include <linux/units.h> -#include <drm/drm_atomic_helper.h> #include <drm/drm_client.h> #include <drm/drm_gem_ttm_helper.h> #include <drm/drm_ioctl.h> #include <drm/drm_managed.h> #include <drm/drm_pagemap_util.h> #include <drm/drm_print.h> +#include <kunit/static_stub.h> #include <uapi/drm/xe_drm.h> #include "display/xe_display.h" @@ -25,6 +25,7 @@ #include "regs/xe_regs.h" #include "xe_bo.h" #include "xe_bo_evict.h" +#include "xe_configfs.h" #include "xe_debugfs.h" #include "xe_defaults.h" #include "xe_devcoredump.h" @@ -65,6 +66,7 @@ #include "xe_survivability_mode.h" #include "xe_sriov.h" #include "xe_svm.h" +#include "xe_sysctrl.h" #include "xe_tile.h" #include "xe_ttm_stolen_mgr.h" #include "xe_ttm_sys_mgr.h" @@ -389,8 +391,9 @@ bool xe_is_xe_file(const struct file *file) return file->f_op == &xe_driver_fops; } -static struct drm_driver driver = { +static const struct drm_driver regular_driver = { .driver_features = + XE_DISPLAY_DRIVER_FEATURES | DRIVER_GEM | DRIVER_RENDER | DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA, @@ -412,8 +415,45 @@ static struct drm_driver driver = { .major = DRIVER_MAJOR, .minor = DRIVER_MINOR, .patchlevel = DRIVER_PATCHLEVEL, + XE_DISPLAY_DRIVER_OPS, }; +#ifdef CONFIG_PCI_IOV +static const struct drm_ioctl_desc xe_ioctls_admin_only[] = { + DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), +}; + +static const struct drm_driver admin_only_driver = { + .driver_features = + XE_DISPLAY_DRIVER_FEATURES | + DRIVER_GEM | DRIVER_RENDER | DRIVER_GEM_GPUVA, + .open = xe_file_open, + .postclose = xe_file_close, + .ioctls = xe_ioctls_admin_only, + .num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only), + .fops = &xe_driver_fops, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, + XE_DISPLAY_DRIVER_OPS, +}; + +/** + * xe_device_is_admin_only() - Check whether device is admin only or not. + * @xe: the &xe_device to check + * + * Return: true if the device is admin only, false otherwise. + */ +bool xe_device_is_admin_only(const struct xe_device *xe) +{ + KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe); + return xe->drm.driver == &admin_only_driver; +} +#endif + static void xe_device_destroy(struct drm_device *dev, void *dummy) { struct xe_device *xe = to_xe_device(dev); @@ -438,16 +478,24 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy) struct xe_device *xe_device_create(struct pci_dev *pdev, const struct pci_device_id *ent) { + const struct drm_driver *driver = ®ular_driver; struct xe_device *xe; int err; - xe_display_driver_set_hooks(&driver); +#ifdef CONFIG_PCI_IOV + /* + * Since XE device is not initialized yet, read from configfs + * directly to decide whether we are in admin-only PF mode or not. + */ + if (xe_configfs_admin_only_pf(pdev)) + driver = &admin_only_driver; +#endif - err = aperture_remove_conflicting_pci_devices(pdev, driver.name); + err = aperture_remove_conflicting_pci_devices(pdev, driver->name); if (err) return ERR_PTR(err); - xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm); + xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm); if (IS_ERR(xe)) return xe; @@ -707,6 +755,11 @@ int xe_device_probe_early(struct xe_device *xe) xe_sriov_probe_early(xe); + if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) { + xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n"); + return -ENODEV; + } + if (IS_SRIOV_VF(xe)) vf_update_device_info(xe); @@ -992,6 +1045,10 @@ int xe_device_probe(struct xe_device *xe) if (err) goto err_unregister_display; + err = xe_sysctrl_init(xe); + if (err) + goto err_unregister_display; + err = xe_device_sysfs_init(xe); if (err) goto err_unregister_display; diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index e4b9de8d8e95..355d69dc8f54 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -146,37 +146,37 @@ static inline struct xe_force_wake *gt_to_fw(struct xe_gt *gt) void xe_device_assert_mem_access(struct xe_device *xe); -static inline bool xe_device_has_flat_ccs(struct xe_device *xe) +static inline bool xe_device_has_flat_ccs(const struct xe_device *xe) { return xe->info.has_flat_ccs; } -static inline bool xe_device_has_sriov(struct xe_device *xe) +static inline bool xe_device_has_sriov(const struct xe_device *xe) { return xe->info.has_sriov; } -static inline bool xe_device_has_msix(struct xe_device *xe) +static inline bool xe_device_has_msix(const struct xe_device *xe) { return xe->irq.msix.nvec > 0; } -static inline bool xe_device_has_memirq(struct xe_device *xe) +static inline bool xe_device_has_memirq(const struct xe_device *xe) { return GRAPHICS_VERx100(xe) >= 1250; } -static inline bool xe_device_uses_memirq(struct xe_device *xe) +static inline bool xe_device_uses_memirq(const struct xe_device *xe) { return xe_device_has_memirq(xe) && (IS_SRIOV_VF(xe) || xe_device_has_msix(xe)); } -static inline bool xe_device_has_lmtt(struct xe_device *xe) +static inline bool xe_device_has_lmtt(const struct xe_device *xe) { return IS_DGFX(xe); } -static inline bool xe_device_has_mert(struct xe_device *xe) +static inline bool xe_device_has_mert(const struct xe_device *xe) { return xe->info.has_mert; } @@ -211,6 +211,15 @@ bool xe_is_xe_file(const struct file *file); struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid); +#ifdef CONFIG_PCI_IOV +bool xe_device_is_admin_only(const struct xe_device *xe); +#else +static inline bool xe_device_is_admin_only(const struct xe_device *xe) +{ + return false; +} +#endif + /* * Occasionally it is seen that the G2H worker starts running after a delay of more than * a second even after being queued and activated by the Linux workqueue subsystem. This diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 615218d775b1..32dd2ffbc796 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -27,6 +27,7 @@ #include "xe_sriov_vf_ccs_types.h" #include "xe_step_types.h" #include "xe_survivability_mode_types.h" +#include "xe_sysctrl_types.h" #include "xe_tile_types.h" #include "xe_validation.h" @@ -41,6 +42,7 @@ struct xe_ggtt; struct xe_i2c; struct xe_pat_ops; struct xe_pxp; +struct xe_ttm_stolen_mgr; struct xe_vram_region; /** @@ -196,6 +198,8 @@ struct xe_device { u8 has_soc_remapper_telem:1; /** @info.has_sriov: Supports SR-IOV */ u8 has_sriov:1; + /** @info.has_sysctrl: Supports System Controller */ + u8 has_sysctrl:1; /** @info.has_usm: Device has unified shared memory support */ u8 has_usm:1; /** @info.has_64bit_timestamp: Device supports 64-bit timestamps */ @@ -215,8 +219,6 @@ struct xe_device { u8 probe_display:1; /** @info.skip_guc_pc: Skip GuC based PM feature init */ u8 skip_guc_pc:1; - /** @info.skip_mtcfg: skip Multi-Tile configuration from MTCFG register */ - u8 skip_mtcfg:1; /** @info.skip_pcode: skip access to PCODE uC */ u8 skip_pcode:1; /** @info.needs_shared_vf_gt_wq: needs shared GT WQ on VF */ @@ -273,8 +275,10 @@ struct xe_device { struct xe_vram_region *vram; /** @mem.sys_mgr: system TTM manager */ struct ttm_resource_manager sys_mgr; - /** @mem.sys_mgr: system memory shrinker. */ + /** @mem.shrinker: system memory shrinker. */ struct xe_shrinker *shrinker; + /** @mem.stolen_mgr: stolen memory manager. */ + struct xe_ttm_stolen_mgr *stolen_mgr; } mem; /** @sriov: device level virtualization data */ @@ -295,7 +299,7 @@ struct xe_device { /** @usm: unified memory state */ struct { - /** @usm.asid: convert a ASID to VM */ + /** @usm.asid_to_vm: convert an ASID to VM */ struct xarray asid_to_vm; /** @usm.next_asid: next ASID, used to cyclical alloc asids */ u32 next_asid; @@ -312,7 +316,7 @@ struct xe_device { /** @usm.pf_queue: Page fault queues */ struct xe_pagefault_queue pf_queue[XE_PAGEFAULT_QUEUE_COUNT]; #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) - /** @usm.pagemap_shrinker: Shrinker for unused pagemaps */ + /** @usm.dpagemap_shrinker: Shrinker for unused pagemaps */ struct drm_pagemap_shrinker *dpagemap_shrinker; #endif } usm; @@ -334,7 +338,7 @@ struct xe_device { struct list_head kernel_bo_present; /** @pinned.late.evicted: pinned BO that have been evicted */ struct list_head evicted; - /** @pinned.external: pinned external and dma-buf. */ + /** @pinned.late.external: pinned external and dma-buf. */ struct list_head external; } late; } pinned; @@ -369,7 +373,7 @@ struct xe_device { struct { /** * @mem_access.vram_userfault.lock: Protects access to - * @vram_usefault.list Using mutex instead of spinlock + * @mem_access.vram_userfault.list Using mutex instead of spinlock * as lock is applied to entire list operation which * may sleep */ @@ -400,7 +404,7 @@ struct xe_device { const struct xe_pat_table_entry *pat_primary_pta; /** @pat.pat_media_pta: media GT PAT entry for page table accesses */ const struct xe_pat_table_entry *pat_media_pta; - u32 idx[__XE_CACHE_LEVEL_COUNT]; + u16 idx[__XE_CACHE_LEVEL_COUNT]; } pat; /** @d3cold: Encapsulate d3cold related stuff */ @@ -508,6 +512,9 @@ struct xe_device { /** @i2c: I2C host controller */ struct xe_i2c *i2c; + /** @sc: System Controller */ + struct xe_sysctrl sc; + /** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */ u32 atomic_svm_timeslice_ms; @@ -578,7 +585,7 @@ struct xe_file { /** @vm: VM state for file */ struct { - /** @vm.xe: xarray to store VMs */ + /** @vm.xa: xarray to store VMs */ struct xarray xa; /** * @vm.lock: Protects VM lookup + reference and removal from diff --git a/drivers/gpu/drm/xe/xe_device_wa_oob.rules b/drivers/gpu/drm/xe/xe_device_wa_oob.rules index d129cddb6ead..d8dc41851425 100644 --- a/drivers/gpu/drm/xe/xe_device_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_device_wa_oob.rules @@ -4,3 +4,6 @@ 22019338487_display PLATFORM(LUNARLAKE) 14022085890 SUBPLATFORM(BATTLEMAGE, G21) 14026539277 PLATFORM(NOVALAKE_P), PLATFORM_STEP(A0, B0) +14026633728 PLATFORM(CRESCENTISLAND) +14026746987 PLATFORM(CRESCENTISLAND) +14026779378 PLATFORM(CRESCENTISLAND) diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index 7f9602b3363d..8a920e58245c 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -193,6 +193,18 @@ static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, return 0; } +static void xe_dma_buf_release(struct dma_buf *dmabuf) +{ + struct drm_gem_object *obj = dmabuf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + + xe_bo_lock(bo, false); + xe_bo_willneed_put_locked(bo); + xe_bo_unlock(bo); + + drm_gem_dmabuf_release(dmabuf); +} + static const struct dma_buf_ops xe_dmabuf_ops = { .attach = xe_dma_buf_attach, .detach = xe_dma_buf_detach, @@ -200,7 +212,7 @@ static const struct dma_buf_ops xe_dmabuf_ops = { .unpin = xe_dma_buf_unpin, .map_dma_buf = xe_dma_buf_map, .unmap_dma_buf = xe_dma_buf_unmap, - .release = drm_gem_dmabuf_release, + .release = xe_dma_buf_release, .begin_cpu_access = xe_dma_buf_begin_cpu_access, .mmap = drm_gem_dmabuf_mmap, .vmap = drm_gem_dmabuf_vmap, @@ -241,26 +253,33 @@ struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) ret = -EINVAL; goto out_unlock; } + + xe_bo_willneed_get_locked(bo); xe_bo_unlock(bo); ret = ttm_bo_setup_export(&bo->ttm, &ctx); if (ret) - return ERR_PTR(ret); + goto out_put; buf = drm_gem_prime_export(obj, flags); - if (!IS_ERR(buf)) - buf->ops = &xe_dmabuf_ops; + if (IS_ERR(buf)) { + ret = PTR_ERR(buf); + goto out_put; + } + buf->ops = &xe_dmabuf_ops; return buf; +out_put: + xe_bo_lock(bo, false); + xe_bo_willneed_put_locked(bo); out_unlock: xe_bo_unlock(bo); return ERR_PTR(ret); } static struct drm_gem_object * -xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, - struct dma_buf *dma_buf) +xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) { struct dma_resv *resv = dma_buf->resv; struct xe_device *xe = to_xe_device(dev); @@ -281,7 +300,7 @@ xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, if (ret) break; - bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size, + bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size, 0, /* Will require 1way or 2way for vm_bind */ ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec); drm_exec_retry_on_contention(&exec); @@ -332,7 +351,6 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, const struct dma_buf_attach_ops *attach_ops; struct dma_buf_attachment *attach; struct drm_gem_object *obj; - struct xe_bo *bo; if (dma_buf->ops == &xe_dmabuf_ops) { obj = dma_buf->priv; @@ -348,13 +366,15 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, } /* - * Don't publish the bo until we have a valid attachment, and a - * valid attachment needs the bo address. So pre-create a bo before - * creating the attachment and publish. + * This needs to happen before the attach, since it will create a new + * attachment for this, and add it to the list of attachments, at which + * point it is globally visible, and at any point the export side can + * call into on invalidate_mappings callback, which require a working + * object. */ - bo = xe_bo_alloc(); - if (IS_ERR(bo)) - return ERR_CAST(bo); + obj = xe_dma_buf_create_obj(dev, dma_buf); + if (IS_ERR(obj)) + return obj; attach_ops = &xe_dma_buf_attach_ops; #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) @@ -362,26 +382,15 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, attach_ops = test->attach_ops; #endif - attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base); + attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, obj); if (IS_ERR(attach)) { - obj = ERR_CAST(attach); - goto out_err; + xe_bo_put(gem_to_xe_bo(obj)); + return ERR_CAST(attach); } - /* Errors here will take care of freeing the bo. */ - obj = xe_dma_buf_init_obj(dev, bo, dma_buf); - if (IS_ERR(obj)) - return obj; - - get_dma_buf(dma_buf); obj->import_attach = attach; return obj; - -out_err: - xe_bo_free(bo); - - return obj; } #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c index e07dc23a155e..c21c8b428de6 100644 --- a/drivers/gpu/drm/xe/xe_drm_ras.c +++ b/drivers/gpu/drm/xe/xe_drm_ras.c @@ -27,6 +27,16 @@ static int hw_query_error_counter(struct xe_drm_ras_counter *info, return 0; } +static int hw_clear_error_counter(struct xe_drm_ras_counter *info, u32 error_id) +{ + if (!info || !info[error_id].name) + return -ENOENT; + + atomic_set(&info[error_id].counter, 0); + + return 0; +} + static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_id, const char **name, u32 *val) { @@ -37,6 +47,15 @@ static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_ return hw_query_error_counter(info, error_id, name, val); } +static int clear_uncorrectable_error_counter(struct drm_ras_node *node, u32 error_id) +{ + struct xe_device *xe = node->priv; + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_UNCORRECTABLE]; + + return hw_clear_error_counter(info, error_id); +} + static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id, const char **name, u32 *val) { @@ -47,6 +66,15 @@ static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id return hw_query_error_counter(info, error_id, name, val); } +static int clear_correctable_error_counter(struct drm_ras_node *node, u32 error_id) +{ + struct xe_device *xe = node->priv; + struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE]; + + return hw_clear_error_counter(info, error_id); +} + static struct xe_drm_ras_counter *allocate_and_copy_counters(struct xe_device *xe) { struct xe_drm_ras_counter *counter; @@ -92,10 +120,13 @@ static int assign_node_params(struct xe_device *xe, struct drm_ras_node *node, if (IS_ERR(ras->info[severity])) return PTR_ERR(ras->info[severity]); - if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) + if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) { node->query_error_counter = query_correctable_error_counter; - else + node->clear_error_counter = clear_correctable_error_counter; + } else { node->query_error_counter = query_uncorrectable_error_counter; + node->clear_error_counter = clear_uncorrectable_error_counter; + } return 0; } diff --git a/drivers/gpu/drm/xe/xe_drm_ras.h b/drivers/gpu/drm/xe/xe_drm_ras.h index 5cc8f0124411..365c70e93e82 100644 --- a/drivers/gpu/drm/xe/xe_drm_ras.h +++ b/drivers/gpu/drm/xe/xe_drm_ras.h @@ -2,8 +2,8 @@ /* * Copyright © 2026 Intel Corporation */ -#ifndef XE_DRM_RAS_H_ -#define XE_DRM_RAS_H_ +#ifndef _XE_DRM_RAS_H_ +#define _XE_DRM_RAS_H_ struct xe_device; diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index c34408cfd292..297be3c42b20 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -44,6 +44,7 @@ struct per_xecore_buf { struct xe_eu_stall_data_stream { bool pollin; bool enabled; + bool reset_detected; int wait_num_reports; int sampling_rate_mult; wait_queue_head_t poll_wq; @@ -428,9 +429,20 @@ static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream) set_bit(xecore, stream->data_drop.mask); xecore_buf->write = write_ptr; } + /* If a GT or engine reset happens during EU stall sampling, + * all EU stall registers get reset to 0 and the cached values of + * the EU stall data buffers' read pointers are out of sync with + * the register values. This causes invalid data to be returned + * from read(). To prevent this, check the value of a EU stall base + * register. If it is zero, there has been a reset. + */ + if (unlikely(!xe_gt_mcr_unicast_read_any(gt, XEHPC_EUSTALL_BASE))) + stream->reset_detected = true; + + stream->pollin = min_data_present || stream->reset_detected; mutex_unlock(&stream->xecore_buf_lock); - return min_data_present; + return stream->pollin; } static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 instance) @@ -544,6 +556,15 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st int ret = 0; mutex_lock(&stream->xecore_buf_lock); + /* If EU stall registers got reset due to a GT/engine reset, + * continuing with the read() will return invalid data to + * the user space. Just return -ENODEV instead. + */ + if (unlikely(stream->reset_detected)) { + xe_gt_dbg(gt, "EU stall base register has been reset\n"); + mutex_unlock(&stream->xecore_buf_lock); + return -ENODEV; + } if (bitmap_weight(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS)) { if (!stream->data_drop.reported_to_user) { stream->data_drop.reported_to_user = true; @@ -554,7 +575,6 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st } stream->data_drop.reported_to_user = false; } - for_each_dss_steering(xecore, gt, group, instance) { ret = xe_eu_stall_data_buf_read(stream, buf, count, &total_size, gt, group, instance, xecore); @@ -609,7 +629,8 @@ static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, * We don't want to block the next read() when there is data in the buffer * now, but couldn't be accommodated in the small user buffer. */ - stream->pollin = false; + if (!stream->reset_detected) + stream->pollin = false; return ret; } @@ -692,6 +713,7 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream) xecore_buf->write = write_ptr; xecore_buf->read = write_ptr; } + stream->reset_detected = false; stream->data_drop.reported_to_user = false; bitmap_zero(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS); @@ -717,13 +739,13 @@ static void eu_stall_data_buf_poll_work_fn(struct work_struct *work) container_of(work, typeof(*stream), buf_poll_work.work); struct xe_gt *gt = stream->gt; - if (eu_stall_data_buf_poll(stream)) { - stream->pollin = true; + if (eu_stall_data_buf_poll(stream)) wake_up(&stream->poll_wq); - } - queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, - &stream->buf_poll_work, - msecs_to_jiffies(POLL_PERIOD_MS)); + + if (!stream->reset_detected) + queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq, + &stream->buf_poll_work, + msecs_to_jiffies(POLL_PERIOD_MS)); } static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream, @@ -869,14 +891,14 @@ static int xe_eu_stall_stream_close(struct inode *inode, struct file *file) struct xe_eu_stall_data_stream *stream = file->private_data; struct xe_gt *gt = stream->gt; - drm_dev_put(>->tile->xe->drm); - mutex_lock(>->eu_stall->stream_lock); xe_eu_stall_disable_locked(stream); xe_eu_stall_data_buf_destroy(stream); xe_eu_stall_stream_free(stream); mutex_unlock(>->eu_stall->stream_lock); + drm_dev_put(>->tile->xe->drm); + return 0; } diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h index d1c76e503799..842bef9f6872 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.h +++ b/drivers/gpu/drm/xe/xe_eu_stall.h @@ -3,8 +3,8 @@ * Copyright © 2025 Intel Corporation */ -#ifndef __XE_EU_STALL_H__ -#define __XE_EU_STALL_H__ +#ifndef _XE_EU_STALL_H_ +#define _XE_EU_STALL_H_ #include "xe_gt_types.h" #include "xe_sriov.h" diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index b287d0e0e60a..1b5ca3ce578a 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -275,8 +275,12 @@ static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u { xe_assert(gt_to_xe(q->gt), idx < q->width); - scoped_guard(spinlock, &q->lrc_lookup_lock) + scoped_guard(spinlock, &q->lrc_lookup_lock) { q->lrc[idx] = lrc; + if (xe_exec_queue_is_multi_queue(q)) + q->lrc[idx]->multi_queue.primary_lrc = + q->multi_queue.group->primary->lrc[0]; + } } /** @@ -852,11 +856,6 @@ static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue * return 0; } -static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q) -{ - return q->gt->info.multi_queue_engine_class_mask & BIT(q->class); -} - static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q, u32 primary_id) { @@ -912,6 +911,7 @@ static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q } q->multi_queue.pos = pos; + q->lrc[0]->multi_queue.pos = pos; return 0; } @@ -931,7 +931,7 @@ static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queu static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q, u64 value) { - if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q))) + if (XE_IOCTL_DBG(xe, !xe_gt_supports_multi_queue(q->gt, q->class))) return -ENODEV; if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe))) @@ -1405,7 +1405,7 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, if (q->vm && q->hwe->hw_engine_group) { err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); if (err) - goto put_exec_queue; + goto kill_exec_queue; } } @@ -1416,12 +1416,15 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, /* user id alloc must always be last in ioctl to prevent UAF */ err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); if (err) - goto kill_exec_queue; + goto del_hw_engine_group; args->exec_queue_id = id; return 0; +del_hw_engine_group: + if (q->vm && q->hwe && q->hwe->hw_engine_group) + xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); kill_exec_queue: xe_exec_queue_kill(q); delete_queue_group: @@ -1760,7 +1763,7 @@ void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q, void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q, unsigned int type) { - xe_assert(q->vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || + xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); dma_fence_put(q->tlb_inval[type].last_fence); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 8ce78e0b1d50..2f5ccf294675 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -214,7 +214,7 @@ struct xe_exec_queue { */ struct xe_dep_scheduler *dep_scheduler; /** - * @last_fence: last fence for tlb invalidation, protected by + * @tlb_inval.last_fence: last fence for tlb invalidation, protected by * vm->lock in write mode */ struct dma_fence *last_fence; diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c index 755a2bff5d7b..9fb99c038ea8 100644 --- a/drivers/gpu/drm/xe/xe_execlist.c +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -47,7 +47,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, struct xe_mmio *mmio = >->mmio; struct xe_device *xe = gt_to_xe(gt); u64 lrc_desc; - u32 ring_mode = REG_MASKED_FIELD_ENABLE(GFX_DISABLE_LEGACY_MODE); lrc_desc = xe_lrc_descriptor(lrc); @@ -59,10 +58,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, lrc_desc |= FIELD_PREP(SW_CTX_ID, ctx_id); } - if (hwe->class == XE_ENGINE_CLASS_COMPUTE) - xe_mmio_write32(mmio, RCU_MODE, - REG_MASKED_FIELD_ENABLE(RCU_MODE_CCS_ENABLE)); - xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); lrc->ring.old_tail = lrc->ring.tail; @@ -82,10 +77,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, xe_bo_ggtt_addr(hwe->hwsp)); xe_mmio_read32(mmio, RING_HWS_PGA(hwe->mmio_base)); - if (xe_device_has_msix(gt_to_xe(hwe->gt))) - ring_mode |= REG_MASKED_FIELD_ENABLE(GFX_MSIX_INTERRUPT_ENABLE); - xe_mmio_write32(mmio, RING_MODE(hwe->mmio_base), ring_mode); - xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_LO(hwe->mmio_base), lower_32_bits(lrc_desc)); xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_HI(hwe->mmio_base), @@ -337,7 +328,6 @@ static int execlist_exec_queue_init(struct xe_exec_queue *q) struct drm_gpu_scheduler *sched; const struct drm_sched_init_args args = { .ops = &drm_sched_ops, - .num_rqs = 1, .credit_limit = xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, .hang_limit = XE_SCHED_HANG_LIMIT, .timeout = XE_SCHED_JOB_TIMEOUT, diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index a848d1a41b9b..a351c578b170 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -24,6 +24,7 @@ #include "xe_gt_types.h" #include "xe_map.h" #include "xe_mmio.h" +#include "xe_pat.h" #include "xe_pm.h" #include "xe_res_cursor.h" #include "xe_sriov.h" @@ -115,7 +116,6 @@ struct xe_ggtt { /** @size: Total usable size of this GGTT */ u64 size; -#define XE_GGTT_FLAGS_64K BIT(0) /** * @flags: Flags for this GGTT * Acceptable flags: @@ -259,7 +259,7 @@ static u64 xe_ggtt_get_pte(struct xe_ggtt *ggtt, u64 addr) static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size) { - u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), XE_CACHE_WB); u64 end = start + size - 1; u64 scratch_pte; @@ -724,7 +724,7 @@ static void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_ggtt_node *node, void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo) { u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB; - u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode]; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode); u64 pte; mutex_lock(&ggtt->lock); @@ -841,7 +841,7 @@ static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, bo->ggtt_node[tile_id] = NULL; } else { u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB; - u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode]; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode); u64 pte = ggtt->pt_ops->pte_encode_flags(bo, pat_index); xe_ggtt_map_bo(ggtt, bo->ggtt_node[tile_id], bo, pte); diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.c b/drivers/gpu/drm/xe/xe_gpu_scheduler.c index 9c8004d5dd91..67d8ce368486 100644 --- a/drivers/gpu/drm/xe/xe_gpu_scheduler.c +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.c @@ -66,7 +66,6 @@ int xe_sched_init(struct xe_gpu_scheduler *sched, const struct drm_sched_init_args args = { .ops = ops, .submit_wq = submit_wq, - .num_rqs = 1, .credit_limit = hw_submission, .hang_limit = hang_limit, .timeout = timeout, diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c index e5c234f3d795..aab59dc647fb 100644 --- a/drivers/gpu/drm/xe/xe_gsc.c +++ b/drivers/gpu/drm/xe/xe_gsc.c @@ -166,7 +166,7 @@ static int query_compatibility_version(struct xe_gsc *gsc) &rd_offset); if (err) { xe_gt_err(gt, "HuC: invalid GSC reply for version query (err=%d)\n", err); - return err; + goto out_bo; } compat->major = version_query_rd(xe, &bo->vmap, rd_offset, proj_major); @@ -482,8 +482,7 @@ int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc) EXEC_QUEUE_FLAG_PERMANENT, 0); if (IS_ERR(q)) { xe_gt_err(gt, "Failed to create queue for GSC submission\n"); - err = PTR_ERR(q); - goto out_bo; + return PTR_ERR(q); } wq = alloc_ordered_workqueue("gsc-ordered-wq", 0); @@ -506,8 +505,6 @@ int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc) out_q: xe_exec_queue_put(q); -out_bo: - xe_bo_unpin_map_no_vm(bo); return err; } diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 8a31c963c372..783eb6d631b5 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -7,6 +7,8 @@ #include <linux/minmax.h> +#include <kunit/visibility.h> + #include <drm/drm_managed.h> #include <uapi/drm/xe_drm.h> @@ -393,10 +395,7 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt) if (gt->default_lrc[hwe->class]) continue; - xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe); - xe_wa_process_lrc(hwe); - xe_hw_engine_setup_default_lrc_state(hwe); - xe_tuning_process_lrc(hwe); + xe_hw_engine_setup_reg_lrc(hwe); default_lrc = drmm_kzalloc(&xe->drm, xe_gt_lrc_size(gt, hwe->class), @@ -788,6 +787,7 @@ void xe_gt_mmio_init(struct xe_gt *gt) if (IS_SRIOV_VF(xe)) gt->mmio.sriov_vf_gt = gt; } +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mmio_init); void xe_gt_record_user_engines(struct xe_gt *gt) { @@ -993,7 +993,6 @@ void xe_gt_reset_async(struct xe_gt *gt) void xe_gt_suspend_prepare(struct xe_gt *gt) { - CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL); xe_uc_suspend_prepare(>->uc); } diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h index de7e47763411..4150aa594f05 100644 --- a/drivers/gpu/drm/xe/xe_gt.h +++ b/drivers/gpu/drm/xe/xe_gt.h @@ -155,4 +155,19 @@ static inline bool xe_gt_recovery_pending(struct xe_gt *gt) xe_gt_sriov_vf_recovery_pending(gt); } +/** + * xe_gt_supports_multi_queue() - Check if gt supports multi queue for the + * specified engine class. + * + * @gt: the GT object + * @class: hwe class type + * + * Return: true if the hw engine class supports multi queue, else false + */ +static inline bool xe_gt_supports_multi_queue(const struct xe_gt *gt, + enum xe_engine_class class) +{ + return gt->info.multi_queue_engine_class_mask & BIT(class); +} + #endif diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index 4a2d9edb6a4c..04b24e1c8b78 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -3,6 +3,8 @@ * Copyright © 2023 Intel Corporation */ +#include <linux/time64.h> + #include <drm/drm_managed.h> #include <generated/xe_wa_oob.h> @@ -93,7 +95,7 @@ static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency) gtidle->cur_residency = cur_residency; /* residency multiplier in ns, convert to ms */ - cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6); + cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, NSEC_PER_MSEC); return cur_residency; } diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c index 7c6f039c880d..04f0098070a4 100644 --- a/drivers/gpu/drm/xe/xe_gt_mcr.c +++ b/drivers/gpu/drm/xe/xe_gt_mcr.c @@ -3,6 +3,9 @@ * Copyright © 2022 Intel Corporation */ +#include <kunit/static_stub.h> +#include <kunit/visibility.h> + #include "xe_gt_mcr.h" #include "regs/xe_gt_regs.h" @@ -216,9 +219,7 @@ static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = { static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = { { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */ - { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */ { 0x00C800, 0x00CFFF }, /* GAM */ - { 0x00D880, 0x00D8FF }, /* NODE */ { 0x00DD00, 0x00DDFF }, /* MEMPIPE */ { 0x00E900, 0x00E97F }, /* MEMPIPE */ { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ @@ -267,7 +268,7 @@ static const struct xe_mmio_range xe3p_xpc_gam_grp1_steering_table[] = { {}, }; -static const struct xe_mmio_range xe3p_xpc_node_steering_table[] = { +static const struct xe_mmio_range xe2_node_steering_table[] = { { 0x00B000, 0x00B0FF }, { 0x00D880, 0x00D8FF }, {}, @@ -298,7 +299,7 @@ static void init_steering_l3bank(struct xe_gt *gt) struct xe_device *xe = gt_to_xe(gt); struct xe_mmio *mmio = >->mmio; - if (GRAPHICS_VER(xe) >= 35) { + if (GRAPHICS_VER(xe) >= 20) { unsigned int first_bank = xe_l3_bank_mask_ffs(gt->fuse_topo.l3_bank_mask); const int banks_per_node = 4; unsigned int node = first_bank / banks_per_node; @@ -430,19 +431,6 @@ void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *g *instance = dss % gt->steering_dss_per_grp; } -/** - * xe_gt_mcr_steering_info_to_dss_id - Get DSS ID from group/instance steering - * @gt: GT structure - * @group: steering group ID - * @instance: steering instance ID - * - * Return: the converted DSS id. - */ -u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance) -{ - return group * dss_per_group(gt) + instance; -} - static void init_steering_dss(struct xe_gt *gt) { gt->steering_dss_per_grp = dss_per_group(gt); @@ -536,7 +524,7 @@ void xe_gt_mcr_init_early(struct xe_gt *gt) gt->steering[GAM1].ranges = xe3p_xpc_gam_grp1_steering_table; gt->steering[INSTANCE0].ranges = xe3p_xpc_instance0_steering_table; gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; - gt->steering[NODE].ranges = xe3p_xpc_node_steering_table; + gt->steering[NODE].ranges = xe2_node_steering_table; } else if (GRAPHICS_VERx100(xe) >= 3510) { gt->steering[DSS].ranges = xe2lpg_dss_steering_table; gt->steering[INSTANCE0].ranges = xe3p_lpg_instance0_steering_table; @@ -544,6 +532,8 @@ void xe_gt_mcr_init_early(struct xe_gt *gt) gt->steering[DSS].ranges = xe2lpg_dss_steering_table; gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table; gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table; + gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; + gt->steering[NODE].ranges = xe2_node_steering_table; } else if (GRAPHICS_VERx100(xe) >= 1270) { gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; @@ -566,6 +556,7 @@ void xe_gt_mcr_init_early(struct xe_gt *gt) /* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */ gt->steering[INSTANCE0].initialized = true; } +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_init_early); /** * xe_gt_mcr_init - Normal initialization of the MCR support @@ -613,6 +604,40 @@ void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) } } +static bool reg_in_steering_type_ranges(struct xe_gt *gt, + struct xe_reg reg, + int type) +{ + if (!gt->steering[type].ranges) + return false; + + for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) + if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) + return true; + + return false; +} + +/* + * xe_gt_mcr_check_reg - check if a register is recognized by this GT as MCR + * @gt: GT structure + * @reg: The register to check + * + * Returns true if the register offset falls within one of the MMIO ranges + * classified as MCR for the GT. + */ +bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg) +{ + KUNIT_STATIC_STUB_REDIRECT(xe_gt_mcr_check_reg, gt, reg); + + for (int type = 0; type <= IMPLICIT_STEERING; type++) + if (reg_in_steering_type_ranges(gt, reg, type)) + return true; + + return false; +} +EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_check_reg); + /* * xe_gt_mcr_get_nonterminated_steering - find group/instance values that * will steer a register to a non-terminated instance @@ -634,30 +659,21 @@ bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, u8 *group, u8 *instance) { const struct xe_reg reg = to_xe_reg(reg_mcr); - const struct xe_mmio_range *implicit_ranges; for (int type = 0; type < IMPLICIT_STEERING; type++) { - if (!gt->steering[type].ranges) - continue; - - for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) { - if (xe_mmio_in_range(>->mmio, >->steering[type].ranges[i], reg)) { - drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized, - "Uninitialized usage of MCR register %s/%#x\n", - xe_steering_types[type].name, reg.addr); - - *group = gt->steering[type].group_target; - *instance = gt->steering[type].instance_target; - return true; - } + if (reg_in_steering_type_ranges(gt, reg, type)) { + drm_WARN(>_to_xe(gt)->drm, !gt->steering[type].initialized, + "Uninitialized usage of MCR register %s/%#x\n", + xe_steering_types[type].name, reg.addr); + + *group = gt->steering[type].group_target; + *instance = gt->steering[type].instance_target; + return true; } } - implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges; - if (implicit_ranges) - for (int i = 0; implicit_ranges[i].end > 0; i++) - if (xe_mmio_in_range(>->mmio, &implicit_ranges[i], reg)) - return false; + if (reg_in_steering_type_ranges(gt, reg, IMPLICIT_STEERING)) + return false; /* * Not found in a steering table and not a register with implicit diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.h b/drivers/gpu/drm/xe/xe_gt_mcr.h index 283a1c9770e2..75374662f10d 100644 --- a/drivers/gpu/drm/xe/xe_gt_mcr.h +++ b/drivers/gpu/drm/xe/xe_gt_mcr.h @@ -26,6 +26,7 @@ void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg, void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg, u32 value); +bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg); bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, u8 *group, u8 *instance); @@ -33,7 +34,6 @@ bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p); void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance); -u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance); /* * Loop over each DSS and determine the group and instance IDs that diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c index fb5c9101e275..7868056841b3 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c @@ -219,6 +219,7 @@ static void pf_restart(struct xe_gt *gt) xe_gt_assert(gt, !xe_pm_runtime_suspended(xe)); + xe_gt_sriov_pf_policy_restart(gt); xe_gt_sriov_pf_config_restart(gt); xe_gt_sriov_pf_control_restart(gt); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 2f376b5fb088..2c9b85b84b1b 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -285,6 +285,13 @@ static u32 encode_config_ggtt(u32 *cfg, const struct xe_gt_sriov_config *config, return encode_ggtt(cfg, xe_ggtt_node_addr(node), xe_ggtt_node_size(node), details); } +static bool custom_sched_priority(struct xe_gt *gt, u32 priority) +{ + return xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt) ? + priority != GUC_SCHED_PRIORITY_NORMAL : + priority != GUC_SCHED_PRIORITY_LOW; +} + static u32 encode_config_sched(struct xe_gt *gt, u32 *cfg, u32 n, const struct xe_gt_sriov_config *config) { @@ -313,6 +320,11 @@ static u32 encode_config_sched(struct xe_gt *gt, u32 *cfg, u32 n, cfg[n++] = config->preempt_timeout[0]; } + if (custom_sched_priority(gt, config->sched_priority)) { + cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_SCHED_PRIORITY); + cfg[n++] = config->sched_priority; + } + return n; } @@ -1922,29 +1934,45 @@ static u64 pf_profile_fair_lmem(struct xe_gt *gt, unsigned int num_vfs) return ALIGN_DOWN(fair, alignment); } -static void __pf_show_provisioning_lmem(struct xe_gt *gt, unsigned int first_vf, - unsigned int num_vfs, bool provisioned) +static void __pf_show_provisioned(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned, + u32 (*get32)(struct xe_gt *, unsigned int), + u64 (*get64)(struct xe_gt *, unsigned int), + const char *what) { unsigned int allvfs = 1 + xe_gt_sriov_pf_get_totalvfs(gt); /* PF plus VFs */ unsigned long *bitmap __free(bitmap) = bitmap_zalloc(allvfs, GFP_KERNEL); unsigned int weight; unsigned int n; + bool pf; + + xe_gt_assert(gt, get32 || get64); if (!bitmap) return; for (n = first_vf; n < first_vf + num_vfs; n++) { - if (!!pf_get_vf_config_lmem(gt, VFID(n)) == provisioned) + if ((get32 && (!!get32(gt, VFID(n)) == provisioned)) || + (get64 && (!!get64(gt, VFID(n)) == provisioned))) bitmap_set(bitmap, n, 1); } + pf = test_and_clear_bit(0, bitmap); weight = bitmap_weight(bitmap, allvfs); - if (!weight) + if (!pf && !weight) return; - xe_gt_sriov_info(gt, "VF%s%*pbl %s provisioned with VRAM\n", - weight > 1 ? "s " : "", allvfs, bitmap, - provisioned ? "already" : "not"); + xe_gt_sriov_info(gt, "%s%s%s%s%*pbl %s provisioned with %s\n", + pf ? "PF" : "", pf && weight ? " and " : "", + weight ? "VF" : "", weight > 1 ? "s " : "", + allvfs, bitmap, provisioned ? "already" : "not", what); +} + +static void __pf_show_provisioning_lmem(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned) +{ + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + NULL, pf_get_vf_config_lmem, "VRAM"); } static void pf_show_all_provisioned_lmem(struct xe_gt *gt) @@ -2078,6 +2106,17 @@ static u32 pf_get_exec_quantum(struct xe_gt *gt, unsigned int vfid) return config->exec_quantum[0]; } +static bool pf_non_default_exec_quantum(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int i; + + for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++) + if (config->exec_quantum[i]) + return true; + return false; +} + /** * xe_gt_sriov_pf_config_set_exec_quantum_locked() - Configure PF/VF execution quantum. * @gt: the &xe_gt @@ -2154,34 +2193,41 @@ u32 xe_gt_sriov_pf_config_get_exec_quantum(struct xe_gt *gt, unsigned int vfid) return pf_get_exec_quantum(gt, vfid); } -/** - * xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked() - Configure EQ for PF and VFs. - * @gt: the &xe_gt to configure - * @exec_quantum: requested execution quantum in milliseconds (0 is infinity) - * - * This function can only be called on PF with the master mutex hold. - * - * Return: 0 on success or a negative error code on failure. - */ -int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum) +static int pf_bulk_set_exec_quantum(struct xe_gt *gt, u32 exec_quantum, + unsigned int first_vf, unsigned int num_vfs) { - unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); unsigned int n; int err = 0; lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); - for (n = 0; n <= totalvfs; n++) { + for (n = first_vf; n < first_vf + num_vfs; n++) { err = pf_provision_exec_quantum(gt, VFID(n), exec_quantum); if (err) break; } - return pf_config_bulk_set_u32_done(gt, 0, 1 + totalvfs, exec_quantum, + return pf_config_bulk_set_u32_done(gt, first_vf, num_vfs, exec_quantum, pf_get_exec_quantum, "execution quantum", exec_quantum_unit, n, err); } +/** + * xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked() - Configure EQ for PF and VFs. + * @gt: the &xe_gt to configure + * @exec_quantum: requested execution quantum in milliseconds (0 is infinity) + * + * This function can only be called on PF with the master mutex hold. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum) +{ + unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); + + return pf_bulk_set_exec_quantum(gt, exec_quantum, PFID, 1 + totalvfs); +} + static int pf_provision_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid, const u32 *exec_quantums, u32 count) { @@ -2293,6 +2339,17 @@ static u32 pf_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid) return config->preempt_timeout[0]; } +static bool pf_non_default_preempt_timeout(struct xe_gt *gt, unsigned int vfid) +{ + struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); + int i; + + for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++) + if (config->preempt_timeout[i]) + return true; + return false; +} + /** * xe_gt_sriov_pf_config_set_preempt_timeout_locked() - Configure PF/VF preemption timeout. * @gt: the &xe_gt @@ -2368,34 +2425,41 @@ u32 xe_gt_sriov_pf_config_get_preempt_timeout(struct xe_gt *gt, unsigned int vfi return pf_get_preempt_timeout(gt, vfid); } -/** - * xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked() - Configure PT for PF and VFs. - * @gt: the &xe_gt to configure - * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity) - * - * This function can only be called on PF with the master mutex hold. - * - * Return: 0 on success or a negative error code on failure. - */ -int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout) +static int pf_bulk_set_preempt_timeout(struct xe_gt *gt, u32 preempt_timeout, + unsigned int first_vf, unsigned int num_vfs) { - unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); unsigned int n; int err = 0; lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); - for (n = 0; n <= totalvfs; n++) { + for (n = first_vf; n < first_vf + num_vfs; n++) { err = pf_provision_preempt_timeout(gt, VFID(n), preempt_timeout); if (err) break; } - return pf_config_bulk_set_u32_done(gt, 0, 1 + totalvfs, preempt_timeout, + return pf_config_bulk_set_u32_done(gt, first_vf, num_vfs, preempt_timeout, pf_get_preempt_timeout, "preemption timeout", preempt_timeout_unit, n, err); } +/** + * xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked() - Configure PT for PF and VFs. + * @gt: the &xe_gt to configure + * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity) + * + * This function can only be called on PF with the master mutex hold. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout) +{ + unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt); + + return pf_bulk_set_preempt_timeout(gt, preempt_timeout, PFID, 1 + totalvfs); +} + static int pf_provision_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid, const u32 *preempt_timeouts, u32 count) { @@ -2500,7 +2564,7 @@ static int pf_provision_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 return 0; } -static int pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid) +static u32 pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid) { struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid); @@ -2550,6 +2614,35 @@ u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid return priority; } +/** + * xe_gt_sriov_pf_config_force_sched_priority_locked() - Force update scheduling priority. + * @gt: the &xe_gt + * @priority: new scheduling priority to set + * + * This function allows to update cached values of the scheduling priorities of all + * VFs (and PF) as result of applying the `GUC_KLV_VGT_POLICY_SCHED_IF_IDLE`_ policy. + * + * This function can only be called on PF. + */ +void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority) +{ + unsigned int total_vfs = xe_gt_sriov_pf_get_totalvfs(gt); + struct xe_gt_sriov_config *config; + unsigned int n; + + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + for (n = 0; n <= total_vfs; n++) { + config = pf_pick_vf_config(gt, VFID(n)); + config->sched_priority = priority; + } + + pf_config_bulk_set_u32_done(gt, PFID, 1 + total_vfs, priority, + pf_get_sched_priority, "scheduling priority", + sched_priority_unit, n, 0); +} + static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *config) { int i; @@ -2562,6 +2655,104 @@ static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *c } } +static bool pf_non_default_sched(struct xe_gt *gt, unsigned int vfid) +{ + return pf_non_default_exec_quantum(gt, vfid) || + pf_non_default_preempt_timeout(gt, vfid) || + custom_sched_priority(gt, pf_get_sched_priority(gt, vfid)); +} + +static void __pf_show_provisioned_sched(struct xe_gt *gt, unsigned int first_vf, + unsigned int num_vfs, bool provisioned) +{ + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + pf_get_exec_quantum, NULL, "EQ"); + __pf_show_provisioned(gt, first_vf, num_vfs, provisioned, + pf_get_preempt_timeout, NULL, "PT"); + + /* we only care about non-default priorities */ + if (provisioned) + __pf_show_provisioned(gt, first_vf, num_vfs, true, + pf_get_sched_priority, NULL, "PRIORITY"); +} + +static void pf_show_all_provisioned_sched(struct xe_gt *gt) +{ + __pf_show_provisioned_sched(gt, PFID, 1 + xe_gt_sriov_pf_get_totalvfs(gt), true); +} + +static void pf_show_unprovisioned_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + __pf_show_provisioned_sched(gt, PFID, 1 + num_vfs, false); +} + +static bool pf_needs_provision_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + unsigned int vfid; + + for (vfid = PFID; vfid <= PFID + num_vfs; vfid++) { + if (pf_non_default_sched(gt, vfid)) { + pf_show_all_provisioned_sched(gt); + pf_show_unprovisioned_sched(gt, num_vfs); + return false; + } + } + + if (xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt)) { + pf_show_all_provisioned_sched(gt); + pf_show_unprovisioned_sched(gt, num_vfs); + return false; + } + + pf_show_all_provisioned_sched(gt); + return true; +} + +/* With 16ms EQ/PT GuC should be able to handle up to 63 VFs within 2s */ +#define XE_FAIR_EXEC_QUANTUM_MS 16 +#define XE_FAIR_PREEMPT_TIMEOUT_US 16000 +#define XE_FAIR_SCHED_PRIORITY GUC_SCHED_PRIORITY_LOW +#define XE_ADMIN_PF_SCHED_PRIORITY GUC_SCHED_PRIORITY_HIGH + +/** + * xe_gt_sriov_pf_config_set_fair_sched() - Provision PF and VFs with fair scheduling. + * @gt: the &xe_gt + * @num_vfs: number of VFs to provision (can't be 0) + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs) +{ + int result = 0; + int err; + + xe_gt_assert(gt, num_vfs); + xe_gt_assert(gt, XE_FAIR_EXEC_QUANTUM_MS); + xe_gt_assert(gt, XE_FAIR_PREEMPT_TIMEOUT_US); + + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + if (!pf_needs_provision_sched(gt, num_vfs)) + return 0; + + err = pf_bulk_set_exec_quantum(gt, XE_FAIR_EXEC_QUANTUM_MS, PFID, 1 + num_vfs); + result = result ?: err; + err = pf_bulk_set_preempt_timeout(gt, XE_FAIR_PREEMPT_TIMEOUT_US, PFID, 1 + num_vfs); + result = result ?: err; + + xe_gt_assert(gt, XE_FAIR_SCHED_PRIORITY == GUC_SCHED_PRIORITY_LOW); + xe_gt_assert(gt, !xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt)); + + if (xe_sriov_pf_admin_only(gt_to_xe(gt))) { + err = pf_provision_sched_priority(gt, PFID, XE_ADMIN_PF_SCHED_PRIORITY); + result = result ?: err; + } + + return result; +} + static int pf_provision_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index, u32 value) { @@ -2838,6 +3029,9 @@ static int pf_validate_vf_config(struct xe_gt *gt, unsigned int vfid) valid_all = valid_all && valid_lmem; } + /* also check optional EQ/PT/PRIO */ + valid_any = valid_any || pf_non_default_sched(gt, vfid); + return valid_all ? 0 : valid_any ? -ENOKEY : -ENODATA; } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h index 4a004ecd6140..2ec62c12ad5c 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h @@ -71,12 +71,14 @@ int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid); int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority); +void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority); u32 xe_gt_sriov_pf_config_get_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index); int xe_gt_sriov_pf_config_set_threshold(struct xe_gt *gt, unsigned int vfid, enum xe_guc_klv_threshold_index index, u32 value); +int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs); int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs); int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout); int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c index 87a164efcc33..01fe03b9efe8 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c @@ -385,10 +385,10 @@ static int pf_migration_mmio_save(struct xe_gt *gt, unsigned int vfid, void *buf if (xe_gt_is_media_type(gt)) for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) - regs[n] = xe_mmio_read32(>->mmio, MED_VF_SW_FLAG(n)); + regs[n] = xe_mmio_read32(&mmio, MED_VF_SW_FLAG(n)); else for (n = 0; n < VF_SW_FLAG_COUNT; n++) - regs[n] = xe_mmio_read32(>->mmio, VF_SW_FLAG(n)); + regs[n] = xe_mmio_read32(&mmio, VF_SW_FLAG(n)); return 0; } @@ -407,10 +407,10 @@ static int pf_migration_mmio_restore(struct xe_gt *gt, unsigned int vfid, if (xe_gt_is_media_type(gt)) for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) - xe_mmio_write32(>->mmio, MED_VF_SW_FLAG(n), regs[n]); + xe_mmio_write32(&mmio, MED_VF_SW_FLAG(n), regs[n]); else for (n = 0; n < VF_SW_FLAG_COUNT; n++) - xe_mmio_write32(>->mmio, VF_SW_FLAG(n), regs[n]); + xe_mmio_write32(&mmio, VF_SW_FLAG(n), regs[n]); return 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c index 7d532bded02a..a85ba4435378 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c @@ -114,8 +114,10 @@ int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 * VFs with no events are not printed. * * This function can only be called on PF. + * + * Return: always 0 */ -void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p) +int xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p) { unsigned int n, total_vfs = xe_gt_sriov_pf_get_totalvfs(gt); const struct xe_gt_sriov_monitor *data; @@ -144,4 +146,6 @@ void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p #undef __format #undef __value } + + return 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h index 7ca9351a271b..0b8f088d3a16 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h @@ -13,7 +13,7 @@ struct drm_printer; struct xe_gt; void xe_gt_sriov_pf_monitor_flr(struct xe_gt *gt, u32 vfid); -void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p); #ifdef CONFIG_PCI_IOV int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c index 848e24926ecd..e8458d63742d 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c @@ -8,6 +8,7 @@ #include "abi/guc_actions_sriov_abi.h" #include "xe_gt.h" +#include "xe_gt_sriov_pf_config.h" #include "xe_gt_sriov_pf_helpers.h" #include "xe_gt_sriov_pf_policy.h" #include "xe_gt_sriov_printk.h" @@ -16,7 +17,6 @@ #include "xe_guc_ct.h" #include "xe_guc_klv_helpers.h" #include "xe_guc_submit.h" -#include "xe_pm.h" /* * Return: number of KLVs that were successfully parsed and saved, @@ -67,6 +67,15 @@ static int pf_push_policy_buf_klvs(struct xe_gt *gt, u32 num_klvs, return err; } + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) { + struct drm_printer p = xe_gt_dbg_printer(gt); + void *klvs = xe_guc_buf_cpu_ptr(buf); + + xe_gt_sriov_dbg(gt, "pushed policy update with %u KLV%s:\n", + num_klvs, str_plural(num_klvs)); + xe_guc_klv_print(klvs, num_dwords, &p); + } + return 0; } @@ -153,33 +162,14 @@ static int pf_update_policy_u32(struct xe_gt *gt, u16 key, u32 *policy, u32 valu return 0; } -static void pf_bulk_reset_sched_priority(struct xe_gt *gt, u32 priority) -{ - unsigned int total_vfs = 1 + xe_gt_sriov_pf_get_totalvfs(gt); - unsigned int n; - - xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); - lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); - - for (n = 0; n < total_vfs; n++) - gt->sriov.pf.vfs[n].config.sched_priority = priority; -} - static int pf_provision_sched_if_idle(struct xe_gt *gt, bool enable) { - int err; - xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); - err = pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY, - >->sriov.pf.policy.guc.sched_if_idle, - enable); - - if (!err) - pf_bulk_reset_sched_priority(gt, enable ? GUC_SCHED_PRIORITY_NORMAL : - GUC_SCHED_PRIORITY_LOW); - return err; + return pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY, + >->sriov.pf.policy.guc.sched_if_idle, + enable); } static int pf_reprovision_sched_if_idle(struct xe_gt *gt) @@ -187,6 +177,9 @@ static int pf_reprovision_sched_if_idle(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + if (!gt->sriov.pf.policy.guc.sched_if_idle) + return 0; + return pf_provision_sched_if_idle(gt, gt->sriov.pf.policy.guc.sched_if_idle); } @@ -199,7 +192,7 @@ static void pf_sanitize_sched_if_idle(struct xe_gt *gt) } /** - * xe_gt_sriov_pf_policy_set_sched_if_idle - Control the 'sched_if_idle' policy. + * xe_gt_sriov_pf_policy_set_sched_if_idle_locked() - Control the 'sched_if_idle' policy. * @gt: the &xe_gt where to apply the policy * @enable: the value of the 'sched_if_idle' policy * @@ -207,15 +200,60 @@ static void pf_sanitize_sched_if_idle(struct xe_gt *gt) * * Return: 0 on success or a negative error code on failure. */ -int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable) +int xe_gt_sriov_pf_policy_set_sched_if_idle_locked(struct xe_gt *gt, bool enable) { + u32 priority; int err; - mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + err = pf_provision_sched_if_idle(gt, enable); - mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); + if (err) + return err; - return err; + /* + * As of GuC 70.12 a change of this policy impacts individual configs + * of all VFs. See `GUC_KLV_VGT_POLICY_SCHED_IF_IDLE`_ for details. + */ + xe_gt_assert(gt, GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 12)); + + priority = enable ? GUC_SCHED_PRIORITY_NORMAL : GUC_SCHED_PRIORITY_LOW; + xe_gt_sriov_pf_config_force_sched_priority_locked(gt, priority); + + return 0; +} + +/** + * xe_gt_sriov_pf_policy_set_sched_if_idle() - Control the 'sched_if_idle' policy. + * @gt: the &xe_gt where to apply the policy + * @enable: the value of the 'sched_if_idle' policy + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); + + return xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, enable); +} + +/** + * xe_gt_sriov_pf_policy_get_sched_if_idle_locked() - Retrieve value of 'sched_if_idle' policy. + * @gt: the &xe_gt where to read the policy from + * + * This function can only be called on PF. + * + * Return: last value of 'sched_if_idle' policy applied. + */ +bool xe_gt_sriov_pf_policy_get_sched_if_idle_locked(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + + return gt->sriov.pf.policy.guc.sched_if_idle; } /** @@ -228,15 +266,10 @@ int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable) */ bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt) { - bool enable; - xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); - mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); - enable = gt->sriov.pf.policy.guc.sched_if_idle; - mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); - - return enable; + return xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt); } static int pf_provision_reset_engine(struct xe_gt *gt, bool enable) @@ -253,6 +286,9 @@ static int pf_reprovision_reset_engine(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + if (!gt->sriov.pf.policy.guc.reset_engine) + return 0; + return pf_provision_reset_engine(gt, gt->sriov.pf.policy.guc.reset_engine); } @@ -319,6 +355,9 @@ static int pf_reprovision_sample_period(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt)); + if (!gt->sriov.pf.policy.guc.sample_period) + return 0; + return pf_provision_sample_period(gt, gt->sriov.pf.policy.guc.sample_period); } @@ -689,30 +728,23 @@ void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt) } /** - * xe_gt_sriov_pf_policy_reprovision - Reprovision (and optionally reset) policy settings. + * xe_gt_sriov_pf_policy_restart() - Reprovision policy settings. * @gt: the &xe_gt - * @reset: if true will reprovision using default values instead of latest * * This function can only be called on PF. * * Return: 0 on success or a negative error code on failure. */ -int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset) +int xe_gt_sriov_pf_policy_restart(struct xe_gt *gt) { int err = 0; - xe_pm_runtime_get_noresume(gt_to_xe(gt)); + guard(mutex)(xe_gt_sriov_pf_master_mutex(gt)); - mutex_lock(xe_gt_sriov_pf_master_mutex(gt)); - if (reset) - pf_sanitize_guc_policies(gt); err |= pf_reprovision_sched_if_idle(gt); err |= pf_reprovision_reset_engine(gt); err |= pf_reprovision_sample_period(gt); err |= pf_reprovision_sched_groups(gt); - mutex_unlock(xe_gt_sriov_pf_master_mutex(gt)); - - xe_pm_runtime_put(gt_to_xe(gt)); return err ? -ENXIO : 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h index bd73aa58f9ca..d667d66897b1 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h @@ -14,7 +14,9 @@ struct drm_printer; struct xe_gt; int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable); +int xe_gt_sriov_pf_policy_set_sched_if_idle_locked(struct xe_gt *gt, bool enable); bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt); +bool xe_gt_sriov_pf_policy_get_sched_if_idle_locked(struct xe_gt *gt); int xe_gt_sriov_pf_policy_set_reset_engine(struct xe_gt *gt, bool enable); bool xe_gt_sriov_pf_policy_get_reset_engine(struct xe_gt *gt); int xe_gt_sriov_pf_policy_set_sample_period(struct xe_gt *gt, u32 value); @@ -29,7 +31,7 @@ bool xe_gt_sriov_pf_policy_sched_groups_enabled(struct xe_gt *gt); void xe_gt_sriov_pf_policy_init(struct xe_gt *gt); void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt); -int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset); +int xe_gt_sriov_pf_policy_restart(struct xe_gt *gt); int xe_gt_sriov_pf_policy_print(struct xe_gt *gt, struct drm_printer *p); #endif diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h index 667b8310478d..6296483c44ec 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h @@ -49,7 +49,6 @@ struct xe_gt_sriov_pf_workers { * @service: service data. * @control: control data. * @policy: policy data. - * @migration: migration data. * @spare: PF-only provisioning configuration. * @vfs: metadata for all VFs. */ diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 8989c8e1be95..0cd9d77f3351 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -1137,13 +1137,15 @@ void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val) } /** - * xe_gt_sriov_vf_print_config - Print VF self config. + * xe_gt_sriov_vf_print_config() - Print VF self config. * @gt: the &xe_gt * @p: the &drm_printer * * This function is for VF use only. + * + * Return: always 0. */ -void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p) +int xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p) { struct xe_gt_sriov_vf_selfconfig *config = >->sriov.vf.self_config; struct xe_device *xe = gt_to_xe(gt); @@ -1170,16 +1172,20 @@ void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p) drm_printf(p, "GuC contexts:\t%u\n", config->num_ctxs); drm_printf(p, "GuC doorbells:\t%u\n", config->num_dbs); + + return 0; } /** - * xe_gt_sriov_vf_print_runtime - Print VF's runtime regs received from PF. + * xe_gt_sriov_vf_print_runtime() - Print VF's runtime regs received from PF. * @gt: the &xe_gt * @p: the &drm_printer * * This function is for VF use only. + * + * Return: always 0. */ -void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p) +int xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p) { struct vf_runtime_reg *vf_regs = gt->sriov.vf.runtime.regs; unsigned int size = gt->sriov.vf.runtime.num_regs; @@ -1188,16 +1194,20 @@ void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p) for (; size--; vf_regs++) drm_printf(p, "%#x = %#x\n", vf_regs->offset, vf_regs->value); + + return 0; } /** - * xe_gt_sriov_vf_print_version - Print VF ABI versions. + * xe_gt_sriov_vf_print_version() - Print VF ABI versions. * @gt: the &xe_gt * @p: the &drm_printer * * This function is for VF use only. + * + * Return: always 0. */ -void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p) +int xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p) { struct xe_device *xe = gt_to_xe(gt); struct xe_uc_fw_version *guc_version = >->sriov.vf.guc_version; @@ -1227,6 +1237,8 @@ void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p) GUC_RELAY_VERSION_LATEST_MAJOR, GUC_RELAY_VERSION_LATEST_MINOR); drm_printf(p, "\thandshake:\t%u.%u\n", pf_version->major, pf_version->minor); + + return 0; } static bool vf_post_migration_shutdown(struct xe_gt *gt) diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h index a6f7127521a5..79878f21b1da 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h @@ -35,9 +35,9 @@ bool xe_gt_sriov_vf_sched_groups_enabled(struct xe_gt *gt); u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg); void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val); -void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p); -void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p); -void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p); +int xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p); int xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt); int xe_vf_migration_fixups_complete_count(struct xe_gt *gt); diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c index 59b3b23a54c8..789397514f3e 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats.c +++ b/drivers/gpu/drm/xe/xe_gt_stats.c @@ -9,6 +9,47 @@ #include "xe_device.h" #include "xe_gt_stats.h" +/** + * DOC: Xe GT Statistics + * + * Overview + * ======== + * + * The Xe driver exposes per-GT statistics through the debugfs filesystem at:: + * + * /sys/kernel/debug/dri/<device>/gt<id>/stats + * + * This interface requires the kernel to be built with ``CONFIG_DEBUG_FS=y``. + * + * Reading statistics + * ================== + * + * Reading the file prints all available statistics, one per line, in + * ``name: value`` format:: + * + * $ cat /sys/kernel/debug/dri/0/gt0/stats + * svm_pagefault_count: 0 + * tlb_inval_count: 1234 + * ... + * + * All values are 64-bit unsigned integers aggregated across all CPUs. + * Counters accumulate since the driver was loaded or since the last explicit + * reset. Timing counters use microseconds as their unit; data volume counters + * use KiB. + * + * Resetting statistics + * ==================== + * + * Writing a boolean true value to the file resets all counters to zero:: + * + * echo 1 > /sys/kernel/debug/dri/0/gt0/stats + * + * Any value accepted by ``kstrtobool()`` (e.g. ``1``, ``y``, ``yes``, + * ``on``) triggers the reset. Resetting while the GPU is active may yield + * unpredictable intermediate values; it is recommended to reset only when + * the GPU is idle. + */ + static void xe_gt_stats_fini(struct drm_device *drm, void *arg) { struct xe_gt *gt = arg; diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h index 081c787ddcb6..425491bed6c4 100644 --- a/drivers/gpu/drm/xe/xe_gt_stats_types.h +++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h @@ -8,6 +8,124 @@ #include <linux/types.h> +/** + * enum xe_gt_stats_id - GT statistics identifiers + * @XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT: Total SVM page faults handled. + * @XE_GT_STATS_ID_TLB_INVAL: Total GPU Translation Lookaside Buffer (TLB) + * invalidations issued. + * @XE_GT_STATS_ID_SVM_TLB_INVAL_COUNT: TLB invalidations issued during SVM + * page-fault handling. + * @XE_GT_STATS_ID_SVM_TLB_INVAL_US: Cumulative time (µs) waiting for TLB + * invalidations during SVM page-fault handling. + * + * @XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT: Buffer-object (non-SVM) page faults + * handled. + * @XE_GT_STATS_ID_VMA_PAGEFAULT_KB: Size (KiB) of VMAs involved in + * buffer-object page fault handling. + * @XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT: GPU prefetch faults for + * addresses with no valid backing. + * + * @XE_GT_STATS_ID_SVM_4K_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 4K pages. + * @XE_GT_STATS_ID_SVM_64K_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 64K pages. + * @XE_GT_STATS_ID_SVM_2M_PAGEFAULT_COUNT: SVM page faults resolved by + * mapping 2M pages. + * @XE_GT_STATS_ID_SVM_4K_VALID_PAGEFAULT_COUNT: Valid SVM page faults + * at 4K page size, where the GPU mapping was already valid — resolved without + * creating new mappings. + * @XE_GT_STATS_ID_SVM_64K_VALID_PAGEFAULT_COUNT: Valid SVM page faults at 64K + * page size. + * @XE_GT_STATS_ID_SVM_2M_VALID_PAGEFAULT_COUNT: Valid SVM page faults at 2M + * page size. + * @XE_GT_STATS_ID_SVM_4K_PAGEFAULT_US: Cumulative time (µs) handling 4K SVM + * page faults. + * @XE_GT_STATS_ID_SVM_64K_PAGEFAULT_US: Cumulative time (µs) handling 64K + * SVM page faults. + * @XE_GT_STATS_ID_SVM_2M_PAGEFAULT_US: Cumulative time (µs) handling 2M SVM + * page faults. + * + * @XE_GT_STATS_ID_SVM_4K_MIGRATE_COUNT: 4K pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_64K_MIGRATE_COUNT: 64K pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_2M_MIGRATE_COUNT: 2M pages moved from CPU to device + * memory. + * @XE_GT_STATS_ID_SVM_4K_MIGRATE_US: Cumulative time (µs) moving 4K pages + * from CPU to device memory. + * @XE_GT_STATS_ID_SVM_64K_MIGRATE_US: Cumulative time (µs) moving 64K pages + * from CPU to device memory. + * @XE_GT_STATS_ID_SVM_2M_MIGRATE_US: Cumulative time (µs) moving 2M pages + * from CPU to device memory. + * + * @XE_GT_STATS_ID_SVM_DEVICE_COPY_US: Cumulative time (µs) for memory copies to + * device, across all page sizes. + * @XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_US: Cumulative time (µs) for memory copies + * of 4K pages to device. + * @XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_US: Cumulative time (µs) for memory + * copies of 64K pages to device. + * @XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_US: Cumulative time (µs) for memory copies + * of 2M pages to device. + * @XE_GT_STATS_ID_SVM_CPU_COPY_US: Cumulative time (µs) for memory copies to + * CPU, across all page sizes. + * @XE_GT_STATS_ID_SVM_4K_CPU_COPY_US: Cumulative time (µs) for memory copies of + * 4K pages to CPU. + * @XE_GT_STATS_ID_SVM_64K_CPU_COPY_US: Cumulative time (µs) for memory copies + * of 64K pages to CPU. + * @XE_GT_STATS_ID_SVM_2M_CPU_COPY_US: Cumulative time (µs) for memory copies of + * 2M pages to CPU. + * @XE_GT_STATS_ID_SVM_DEVICE_COPY_KB: Data (KiB) copied to device across all + * page sizes. + * @XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_KB: Data (KiB) copied to device for 4K + * pages. + * @XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_KB: Data (KiB) copied to device for + * 64K pages. + * @XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_KB: Data (KiB) copied to device for 2M + * pages. + * @XE_GT_STATS_ID_SVM_CPU_COPY_KB: Data (KiB) copied to CPU across all page + * sizes. + * @XE_GT_STATS_ID_SVM_4K_CPU_COPY_KB: Data (KiB) copied to CPU for 4K pages. + * @XE_GT_STATS_ID_SVM_64K_CPU_COPY_KB: Data (KiB) copied to CPU for 64K pages. + * @XE_GT_STATS_ID_SVM_2M_CPU_COPY_KB: Data (KiB) copied to CPU for 2M pages. + * + * @XE_GT_STATS_ID_SVM_4K_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 4K page size. + * @XE_GT_STATS_ID_SVM_64K_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 64K page size. + * @XE_GT_STATS_ID_SVM_2M_GET_PAGES_US: Cumulative time (µs) getting CPU + * memory pages for GPU access at 2M page size. + * @XE_GT_STATS_ID_SVM_4K_BIND_US: Cumulative time (µs) binding 4K pages + * into the GPU page table. + * @XE_GT_STATS_ID_SVM_64K_BIND_US: Cumulative time (µs) binding 64K pages + * into the GPU page table. + * @XE_GT_STATS_ID_SVM_2M_BIND_US: Cumulative time (µs) binding 2M pages + * into the GPU page table. + * + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT: Times the + * scheduler preempted a long-running (LR) GPU exec queue. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT: Times the scheduler + * skipped suspend because the system was idle. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT: Times the driver + * stalled waiting for prior GPU work to complete before scheduling more. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US: Cumulative time + * (µs) spent preempting long-running (LR) GPU exec queues. + * @XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US: Cumulative time (µs) + * stalled waiting for prior GPU work to complete. + * + * @XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT: 4K-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT: 64K-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT: 2M-page entries from the page reclaim + * list that were processed. + * @XE_GT_STATS_ID_PRL_ISSUED_COUNT: Times a page reclamation was issued. + * @XE_GT_STATS_ID_PRL_ABORTED_COUNT: Times the page reclaim process was + * aborted. + * + * @__XE_GT_STATS_NUM_IDS: Number of valid IDs; not a real counter. + * + * See Documentation/gpu/xe/xe_gt_stats.rst. + */ enum xe_gt_stats_id { XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT, XE_GT_STATS_ID_TLB_INVAL, diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 8b55cf25a75f..e5588c88800a 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -136,7 +136,7 @@ struct xe_gt { /** @info.gmdid: raw GMD_ID value from hardware */ u32 gmdid; /** - * @multi_queue_engine_class_mask: Bitmask of engine classes with + * @info.multi_queue_engine_class_mask: Bitmask of engine classes with * multi queue support enabled. */ u16 multi_queue_engine_class_mask; @@ -145,6 +145,13 @@ struct xe_gt { /** @info.has_indirect_ring_state: GT has indirect ring state support */ u8 has_indirect_ring_state:1; /** + * @info.has_xe2_blt_instructions: GT supports Xe2-style MEM_SET + * and MEM_COPY blitter functionality. Note that despite the + * name, some Xe1 platforms may also support this "Xe2-style" + * feature. + */ + u8 has_xe2_blt_instructions:1; + /** * @info.num_geometry_xecore_fuse_regs: Number of 32b-bit fuse * registers the geometry XeCore mask spans. */ @@ -355,7 +362,7 @@ struct xe_gt { /** @user_engines: engines present in GT and available to userspace */ struct { /** - * @user_engines.mask: like @info->engine_mask, but take in + * @user_engines.mask: like @info.engine_mask, but take in * consideration only engines available to userspace */ u64 mask; diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c index ccebb437e37f..4023700ff2a9 100644 --- a/drivers/gpu/drm/xe/xe_guc.c +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -98,7 +98,7 @@ static u32 guc_ctl_feature_flags(struct xe_guc *guc) if (xe_guc_using_main_gamctrl_queues(guc)) flags |= GUC_CTL_MAIN_GAMCTRL_QUEUES; - if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe) && xe_gt_is_media_type(guc_to_gt(guc))) + if (xe_device_is_l2_flush_optimized(xe) && xe_gt_is_media_type(guc_to_gt(guc))) flags |= GUC_CTL_ENABLE_L2FLUSH_OPT; return flags; @@ -780,6 +780,14 @@ int xe_guc_init(struct xe_guc *guc) if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 14, 0)) xe->info.has_page_reclaim_hw_assist = false; + /* Disable indirect_ring_state if missing GuC 70.53+ WA 14025515070. */ + if (gt->info.has_indirect_ring_state && + XE_GT_WA(gt, 14025515070) && + GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 26, 0)) { + gt->info.has_indirect_ring_state = 0; + xe_gt_notice(gt, "indirect ring state requires WA in GuC submit ver 1.26+\n"); + } + if (IS_SRIOV_VF(xe)) { ret = devm_add_action_or_reset(xe->drm.dev, vf_guc_fini_hw, guc); if (ret) @@ -1693,13 +1701,8 @@ void xe_guc_reset_wait(struct xe_guc *guc) void xe_guc_stop_prepare(struct xe_guc *guc) { - if (!IS_SRIOV_VF(guc_to_xe(guc))) { - int err; - - err = xe_guc_pc_stop(&guc->pc); - xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n", - ERR_PTR(err)); - } + if (!IS_SRIOV_VF(guc_to_xe(guc))) + xe_guc_pc_stop(&guc->pc); } void xe_guc_stop(struct xe_guc *guc) diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 81b5f01b1f65..b9bca6084a4f 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -360,6 +360,9 @@ static void guc_waklv_init(struct xe_guc_ads *ads) if (XE_GT_WA(gt, 14020001231)) guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT); + if (XE_GT_WA(gt, 14025515070) && GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 53)) + guc_waklv_enable(ads, NULL, 0, &offset, &remain, + GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG); size = guc_ads_waklv_size(ads) - remain; if (!size) @@ -512,12 +515,9 @@ static void guc_golden_lrc_init(struct xe_guc_ads *ads) * that starts after the execlists LRC registers. This is * required to allow the GuC to restore just the engine state * when a watchdog reset occurs. - * We calculate the engine state size by removing the size of - * what comes before it in the context image (which is identical - * on all engines). */ ads_blob_write(ads, ads.eng_state_size[guc_class], - real_size - xe_lrc_skip_size(xe)); + xe_lrc_engine_state_size(gt, class)); ads_blob_write(ads, ads.golden_context_lrca[guc_class], addr_ggtt); @@ -742,10 +742,8 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, struct xe_reg reg; bool skip; } *e, extra_regs[] = { - { .reg = RING_MODE(hwe->mmio_base), }, { .reg = RING_HWS_PGA(hwe->mmio_base), }, { .reg = RING_IMR(hwe->mmio_base), }, - { .reg = RCU_MODE, .skip = hwe != hwe_rcs_reset_domain }, { .reg = CCS_MODE, .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) }, }; diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index 2f5816c78fba..21f7caf9ea08 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -111,7 +111,7 @@ struct __guc_capture_parsed_output { { RING_TAIL(0), REG_32BIT, 0, 0, 0, "RING_TAIL"}, \ { RING_CTL(0), REG_32BIT, 0, 0, 0, "RING_CTL"}, \ { RING_MI_MODE(0), REG_32BIT, 0, 0, 0, "RING_MI_MODE"}, \ - { RING_MODE(0), REG_32BIT, 0, 0, 0, "RING_MODE"}, \ + { GFX_MODE(0), REG_32BIT, 0, 0, 0, "GFX_MODE"}, \ { RING_ESR(0), REG_32BIT, 0, 0, 0, "RING_ESR"}, \ { RING_EMR(0), REG_32BIT, 0, 0, 0, "RING_EMR"}, \ { RING_EIR(0), REG_32BIT, 0, 0, 0, "RING_EIR"}, \ @@ -1841,12 +1841,6 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm str_yes_no(snapshot->kernel_reserved)); for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) { - /* - * FIXME: During devcoredump print we should avoid accessing the - * driver pointers for gt or engine. Printing should be done only - * using the snapshot captured. Here we are accessing the gt - * pointer. It should be fixed. - */ list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, type, capture_class, false); snapshot_print_by_list_order(snapshot, p, type, list); diff --git a/drivers/gpu/drm/xe/xe_guc_capture.h b/drivers/gpu/drm/xe/xe_guc_capture.h index 34d6fdc64f56..dca97d52b192 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.h +++ b/drivers/gpu/drm/xe/xe_guc_capture.h @@ -3,8 +3,8 @@ * Copyright © 2021-2024 Intel Corporation */ -#ifndef _XE_GUC_CAPTURE_H -#define _XE_GUC_CAPTURE_H +#ifndef _XE_GUC_CAPTURE_H_ +#define _XE_GUC_CAPTURE_H_ #include <linux/types.h> #include "abi/guc_capture_abi.h" diff --git a/drivers/gpu/drm/xe/xe_guc_capture_types.h b/drivers/gpu/drm/xe/xe_guc_capture_types.h index 6cb439115597..058a3f2eadce 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture_types.h +++ b/drivers/gpu/drm/xe/xe_guc_capture_types.h @@ -3,8 +3,8 @@ * Copyright © 2021-2024 Intel Corporation */ -#ifndef _XE_GUC_CAPTURE_TYPES_H -#define _XE_GUC_CAPTURE_TYPES_H +#ifndef _XE_GUC_CAPTURE_TYPES_H_ +#define _XE_GUC_CAPTURE_TYPES_H_ #include <linux/types.h> #include "regs/xe_reg_defs.h" @@ -32,7 +32,7 @@ struct __guc_mmio_reg_descr { /** * @data_type: data type of the register * Could be 32 bit, low or hi dword of a 64 bit, see enum - * register_data_type + * capture_register_data_type */ enum capture_register_data_type data_type; /** @flags: Flags for the register */ diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index a11cff7a20be..21e0dad9a481 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -186,13 +186,16 @@ static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { } struct g2h_fence { u32 *response_buffer; u32 seqno; + /* fields below this point are setup based on the response */ u32 response_data; u16 response_len; u16 error; u16 hint; u16 reason; + u32 counter; bool cancel; bool retry; + bool wait; bool fail; bool done; }; @@ -204,6 +207,11 @@ static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) g2h_fence->seqno = ~0x0; } +static void g2h_fence_reinit(struct g2h_fence *g2h_fence) +{ + memset_after(g2h_fence, 0, seqno); +} + static void g2h_fence_cancel(struct g2h_fence *g2h_fence) { g2h_fence->cancel = true; @@ -1331,6 +1339,7 @@ retry_same_fence: /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response * and g2h_fence_cancel. */ +wait_again: ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); if (!ret) { LNL_FLUSH_WORK(&ct->g2h_worker); @@ -1356,6 +1365,14 @@ retry_same_fence: return -ETIME; } + if (g2h_fence.wait) { + xe_gt_dbg(gt, "H2G action %#x busy: counter %u\n", + action[0], g2h_fence.counter); + /* we can't leave any response data if we want to wait again */ + g2h_fence_reinit(&g2h_fence); + mutex_unlock(&ct->lock); + goto wait_again; + } if (g2h_fence.retry) { xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", action[0], g2h_fence.reason); @@ -1508,7 +1525,12 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) return -EPROTO; } - g2h_fence = xa_erase(&ct->fence_lookup, fence); + /* don't erase as we still expect a final response with the same fence */ + if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) + g2h_fence = xa_load(&ct->fence_lookup, fence); + else + g2h_fence = xa_erase(&ct->fence_lookup, fence); + if (unlikely(!g2h_fence)) { /* Don't tear down channel, as send could've timed out */ /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ @@ -1519,6 +1541,12 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) xe_gt_assert(gt, fence == g2h_fence->seqno); + /* + * reinit as we might have already process this g2h_fence before + * if we received a NO_RESPONSE_BUSY reply + */ + g2h_fence_reinit(g2h_fence); + if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { g2h_fence->fail = true; g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); @@ -1526,6 +1554,9 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { g2h_fence->retry = true; g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); + } else if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) { + g2h_fence->wait = true; + g2h_fence->counter = FIELD_GET(GUC_HXG_BUSY_MSG_0_COUNTER, hxg[0]); } else if (g2h_fence->response_buffer) { g2h_fence->response_len = hxg_len; memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); @@ -1533,7 +1564,9 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); } - g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); + /* don't release any space if it was an intermediate message */ + if (!g2h_fence->wait) + g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ WRITE_ONCE(g2h_fence->done, true); @@ -1570,6 +1603,7 @@ static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) case GUC_HXG_TYPE_RESPONSE_SUCCESS: case GUC_HXG_TYPE_RESPONSE_FAILURE: case GUC_HXG_TYPE_NO_RESPONSE_RETRY: + case GUC_HXG_TYPE_NO_RESPONSE_BUSY: ret = parse_g2h_response(ct, msg, len); break; default: diff --git a/drivers/gpu/drm/xe/xe_guc_ct_types.h b/drivers/gpu/drm/xe/xe_guc_ct_types.h index 5da1ce5dc372..f88e588af0d3 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct_types.h +++ b/drivers/gpu/drm/xe/xe_guc_ct_types.h @@ -102,9 +102,9 @@ struct xe_dead_ct { bool reported; /** @worker: worker thread to get out of interrupt context before dumping */ struct work_struct worker; - /** snapshot_ct: copy of CT state and CTB content at point of error */ + /** @snapshot_ct: copy of CT state and CTB content at point of error */ struct xe_guc_ct_snapshot *snapshot_ct; - /** snapshot_log: copy of GuC log at point of error */ + /** @snapshot_log: copy of GuC log at point of error */ struct xe_guc_log_snapshot *snapshot_log; }; @@ -134,9 +134,9 @@ struct xe_guc_ct { spinlock_t fast_lock; /** @ctbs: buffers for sending and receiving commands */ struct { - /** @ctbs.send: Host to GuC (H2G, send) channel */ + /** @ctbs.h2g: Host to GuC (H2G, send) channel */ struct guc_ctb h2g; - /** @ctbs.recv: GuC to Host (G2H, receive) channel */ + /** @ctbs.g2h: GuC to Host (G2H, receive) channel */ struct guc_ctb g2h; } ctbs; /** @g2h_outstanding: number of outstanding G2H */ diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index b73fae063fac..3fbda4798cff 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_GUC_FWIF_H -#define _XE_GUC_FWIF_H +#ifndef _XE_GUC_FWIF_H_ +#define _XE_GUC_FWIF_H_ #include <linux/bits.h> diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index bb8c4e793492..59f2fa79ad42 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -890,9 +890,26 @@ void xe_guc_pc_init_early(struct xe_guc_pc *pc) pc_init_fused_rp_values(pc); } +static bool pc_needs_min_freq_change(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + + if (XE_DEVICE_WA(xe, 14022085890)) + return true; + + if (xe_gt_is_media_type(gt)) + return false; + + if (xe->info.platform == XE_BATTLEMAGE || + xe->info.platform == XE_CRESCENTISLAND) + return true; + + return false; +} + static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) { - struct xe_tile *tile = gt_to_tile(pc_to_gt(pc)); int ret; lockdep_assert_held(&pc->freq_lock); @@ -919,7 +936,18 @@ static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) if (pc_get_min_freq(pc) > pc->rp0_freq) ret = pc_set_min_freq(pc, pc->rp0_freq); - if (XE_DEVICE_WA(tile_to_xe(tile), 14022085890)) + /* + * Setting GT RP min frequency to 1.2GHz by default for + * GT0(Graphics) Tile of BMG and CRI. + * + * While BMG G21 WA will apply min frequency for + * both GT0(Graphics) and GT1(Media) Tile. + * + * This is an active frequency, so if the device is idle + * we aren't expecting high power output across board + * + */ + if (pc_needs_min_freq_change(pc)) ret = pc_set_min_freq(pc, max(BMG_MIN_FREQ, pc_get_min_freq(pc))); out: @@ -1288,18 +1316,16 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) * xe_guc_pc_stop - Stop GuC's Power Conservation component * @pc: Xe_GuC_PC instance */ -int xe_guc_pc_stop(struct xe_guc_pc *pc) +void xe_guc_pc_stop(struct xe_guc_pc *pc) { struct xe_device *xe = pc_to_xe(pc); if (xe->info.skip_guc_pc) - return 0; + return; mutex_lock(&pc->freq_lock); pc->freq_ready = false; mutex_unlock(&pc->freq_lock); - - return 0; } /** @@ -1314,10 +1340,10 @@ static void xe_guc_pc_fini_hw(void *arg) if (xe_device_wedged(xe)) return; - CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FW_GT); - XE_WARN_ON(xe_guc_pc_stop(pc)); + xe_guc_pc_stop(pc); /* Bind requested freq to mert_freq_cap before unload */ + CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FW_GT); pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc))); } diff --git a/drivers/gpu/drm/xe/xe_guc_pc.h b/drivers/gpu/drm/xe/xe_guc_pc.h index 0678a4e787b3..1025a2b15f5f 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.h +++ b/drivers/gpu/drm/xe/xe_guc_pc.h @@ -13,7 +13,7 @@ struct drm_printer; int xe_guc_pc_init(struct xe_guc_pc *pc); int xe_guc_pc_start(struct xe_guc_pc *pc); -int xe_guc_pc_stop(struct xe_guc_pc *pc); +void xe_guc_pc_stop(struct xe_guc_pc *pc); void xe_guc_pc_print(struct xe_guc_pc *pc, struct drm_printer *p); int xe_guc_pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value); int xe_guc_pc_action_unset_param(struct xe_guc_pc *pc, u8 id); diff --git a/drivers/gpu/drm/xe/xe_guc_relay_types.h b/drivers/gpu/drm/xe/xe_guc_relay_types.h index 20eee10856b2..b99a6686416b 100644 --- a/drivers/gpu/drm/xe/xe_guc_relay_types.h +++ b/drivers/gpu/drm/xe/xe_guc_relay_types.h @@ -15,7 +15,7 @@ * struct xe_guc_relay - Data used by the VF-PF Relay Communication over GuC. */ struct xe_guc_relay { - /**@lock: protects all internal data. */ + /** @lock: protects all internal data. */ spinlock_t lock; /** @worker: dispatches incoming action messages. */ diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index a145234f662b..afd8cc7bd231 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -261,22 +261,10 @@ static void guc_submit_sw_fini(struct drm_device *drm, void *arg) static void guc_submit_fini(void *arg) { struct xe_guc *guc = arg; - - /* Forcefully kill any remaining exec queues */ - xe_guc_ct_stop(&guc->ct); - guc_submit_reset_prepare(guc); - xe_guc_softreset(guc); - xe_guc_submit_stop(guc); - xe_uc_fw_sanitize(&guc->fw); - xe_guc_submit_pause_abort(guc); -} - -static void guc_submit_wedged_fini(void *arg) -{ - struct xe_guc *guc = arg; struct xe_exec_queue *q; unsigned long index; + /* Drop any wedged queue refs */ mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { if (exec_queue_wedged(q)) { @@ -286,6 +274,14 @@ static void guc_submit_wedged_fini(void *arg) } } mutex_unlock(&guc->submission_state.lock); + + /* Forcefully kill any remaining exec queues */ + xe_guc_ct_stop(&guc->ct); + guc_submit_reset_prepare(guc); + xe_guc_softreset(guc); + xe_guc_submit_stop(guc); + xe_uc_fw_sanitize(&guc->fw); + xe_guc_submit_pause_abort(guc); } static const struct xe_exec_queue_ops guc_exec_queue_ops; @@ -856,10 +852,27 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1); } -static void __register_exec_queue_group(struct xe_guc *guc, - struct xe_exec_queue *q, +static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q) +{ +#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_exec_queue_group *group = q->multi_queue.group; + u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; + int len = 0; + + action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; + action[len++] = group->primary->guc->id; + + xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); +#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE + + xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); +} + +static void __register_exec_queue_group(struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) { + struct xe_guc *guc = exec_queue_to_guc(q); #define MAX_MULTI_QUEUE_REG_SIZE (8) u32 action[MAX_MULTI_QUEUE_REG_SIZE]; int len = 0; @@ -884,29 +897,6 @@ static void __register_exec_queue_group(struct xe_guc *guc, xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); } -static void xe_guc_exec_queue_group_add(struct xe_guc *guc, - struct xe_exec_queue *q) -{ -#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) - u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; - int len = 0; - - xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q)); - - action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; - action[len++] = q->multi_queue.group->primary->guc->id; - - xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); -#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE - - /* - * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a - * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response - * from guc. - */ - xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); -} - static void __register_mlrc_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) @@ -1032,7 +1022,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) set_exec_queue_registered(q); trace_xe_exec_queue_register(q); if (xe_exec_queue_is_multi_queue_primary(q)) - __register_exec_queue_group(guc, q, &info); + __register_exec_queue_group(q, &info); else if (xe_exec_queue_is_parallel(q)) __register_mlrc_exec_queue(guc, q, &info); else if (!xe_exec_queue_is_multi_queue_secondary(q)) @@ -1042,7 +1032,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) init_policies(guc, q); if (xe_exec_queue_is_multi_queue_secondary(q)) - xe_guc_exec_queue_group_add(guc, q); + guc_exec_queue_send_cgp_sync(q); } static u32 wq_space_until_wrap(struct xe_exec_queue *q) @@ -1220,10 +1210,8 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); - if (exec_queue_killed_or_banned_or_wedged(primary)) { - killed_or_banned_or_wedged = true; + if (exec_queue_killed_or_banned_or_wedged(primary)) goto run_job_out; - } if (!exec_queue_registered(primary)) register_exec_queue(primary, GUC_CONTEXT_NORMAL); @@ -1320,10 +1308,8 @@ static void disable_scheduling_deregister(struct xe_guc *guc, void xe_guc_submit_wedge(struct xe_guc *guc) { struct xe_device *xe = guc_to_xe(guc); - struct xe_gt *gt = guc_to_gt(guc); struct xe_exec_queue *q; unsigned long index; - int err; xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); @@ -1335,15 +1321,6 @@ void xe_guc_submit_wedge(struct xe_guc *guc) return; if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { - err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, - guc_submit_wedged_fini, guc); - if (err) { - xe_gt_err(gt, "Failed to register clean-up on wedged.mode=%s; " - "Although device is wedged.\n", - xe_wedged_mode_to_string(XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET)); - return; - } - mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) if (xe_exec_queue_get_unless_zero(q)) @@ -1688,6 +1665,14 @@ static void guc_exec_queue_fini(struct xe_exec_queue *q) struct xe_guc_exec_queue *ge = q->guc; struct xe_guc *guc = exec_queue_to_guc(q); + if (xe_exec_queue_is_multi_queue_secondary(q)) { + struct xe_exec_queue_group *group = q->multi_queue.group; + + mutex_lock(&group->list_lock); + list_del(&q->multi_queue.link); + mutex_unlock(&group->list_lock); + } + release_guc_id(guc, q); xe_sched_entity_fini(&ge->entity); xe_sched_fini(&ge->sched); @@ -1709,14 +1694,6 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w) guard(xe_pm_runtime)(guc_to_xe(guc)); trace_xe_exec_queue_destroy(q); - if (xe_exec_queue_is_multi_queue_secondary(q)) { - struct xe_exec_queue_group *group = q->multi_queue.group; - - mutex_lock(&group->list_lock); - list_del(&q->multi_queue.link); - mutex_unlock(&group->list_lock); - } - /* Confirm no work left behind accessing device structures */ cancel_delayed_work_sync(&ge->sched.base.work_tdr); @@ -1904,21 +1881,8 @@ static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sche { struct xe_exec_queue *q = msg->private_data; - if (guc_exec_queue_allowed_to_change_state(q)) { -#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) - struct xe_guc *guc = exec_queue_to_guc(q); - struct xe_exec_queue_group *group = q->multi_queue.group; - u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; - int len = 0; - - action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; - action[len++] = group->primary->guc->id; - - xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); -#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE - - xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); - } + if (guc_exec_queue_allowed_to_change_state(q)) + guc_exec_queue_send_cgp_sync(q); kfree(msg); } @@ -2983,9 +2947,10 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) if (unlikely(!q)) return -EPROTO; - xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x", - xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id, - atomic_read(&q->guc->state)); + if (!exec_queue_killed(q)) + xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x", + xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id, + atomic_read(&q->guc->state)); trace_xe_exec_queue_reset(q); diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h index 5ccc5f959bb3..7824f61b1290 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit_types.h +++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h @@ -138,7 +138,7 @@ struct xe_guc_submit_exec_queue_snapshot { u32 primary; /** @multi_queue.pos: Position of the exec queue within the multi queue group */ u8 pos; - /** @valid: The exec queue is part of a multi queue group */ + /** @multi_queue.valid: The exec queue is part of a multi queue group */ bool valid; } multi_queue; }; diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 6dd05fac6595..8c66ff6f3d3c 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -67,7 +67,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 0, .irq_offset = ilog2(INTR_BCS(0)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = BLT_RING_BASE, }, [XE_HW_ENGINE_BCS1] = { @@ -75,7 +75,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 1, .irq_offset = ilog2(INTR_BCS(1)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS1_RING_BASE, }, [XE_HW_ENGINE_BCS2] = { @@ -83,7 +83,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 2, .irq_offset = ilog2(INTR_BCS(2)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS2_RING_BASE, }, [XE_HW_ENGINE_BCS3] = { @@ -91,7 +91,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 3, .irq_offset = ilog2(INTR_BCS(3)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS3_RING_BASE, }, [XE_HW_ENGINE_BCS4] = { @@ -99,7 +99,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 4, .irq_offset = ilog2(INTR_BCS(4)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS4_RING_BASE, }, [XE_HW_ENGINE_BCS5] = { @@ -107,7 +107,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 5, .irq_offset = ilog2(INTR_BCS(5)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS5_RING_BASE, }, [XE_HW_ENGINE_BCS6] = { @@ -115,7 +115,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 6, .irq_offset = ilog2(INTR_BCS(6)), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS6_RING_BASE, }, [XE_HW_ENGINE_BCS7] = { @@ -123,7 +123,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .irq_offset = ilog2(INTR_BCS(7)), .instance = 7, - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS7_RING_BASE, }, [XE_HW_ENGINE_BCS8] = { @@ -131,7 +131,7 @@ static const struct engine_info engine_infos[] = { .class = XE_ENGINE_CLASS_COPY, .instance = 8, .irq_offset = ilog2(INTR_BCS8), - .domain = XE_FW_RENDER, + .domain = XE_FW_GT, .mmio_base = XEHPC_BCS8_RING_BASE, }, @@ -283,27 +283,6 @@ static void hw_engine_fini(void *arg) } /** - * xe_hw_engine_mmio_write32() - Write engine register - * @hwe: engine - * @reg: register to write into - * @val: desired 32-bit value to write - * - * This function will write val into an engine specific register. - * Forcewake must be held by the caller. - * - */ -void xe_hw_engine_mmio_write32(struct xe_hw_engine *hwe, - struct xe_reg reg, u32 val) -{ - xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base)); - xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain); - - reg.addr += hwe->mmio_base; - - xe_mmio_write32(&hwe->gt->mmio, reg, val); -} - -/** * xe_hw_engine_mmio_read32() - Read engine register * @hwe: engine * @reg: register to read from @@ -325,24 +304,8 @@ u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg) void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe) { - u32 ccs_mask = - xe_hw_engine_mask_per_class(hwe->gt, XE_ENGINE_CLASS_COMPUTE); - u32 ring_mode = REG_MASKED_FIELD_ENABLE(GFX_DISABLE_LEGACY_MODE); - - if (hwe->class == XE_ENGINE_CLASS_COMPUTE && ccs_mask) - xe_mmio_write32(&hwe->gt->mmio, RCU_MODE, - REG_MASKED_FIELD_ENABLE(RCU_MODE_CCS_ENABLE)); - - xe_hw_engine_mmio_write32(hwe, RING_HWSTAM(0), ~0x0); - xe_hw_engine_mmio_write32(hwe, RING_HWS_PGA(0), - xe_bo_ggtt_addr(hwe->hwsp)); - - if (xe_device_has_msix(gt_to_xe(hwe->gt))) - ring_mode |= REG_MASKED_FIELD_ENABLE(GFX_MSIX_INTERRUPT_ENABLE); - xe_hw_engine_mmio_write32(hwe, RING_MODE(0), ring_mode); - xe_hw_engine_mmio_write32(hwe, RING_MI_MODE(0), - REG_MASKED_FIELD_DISABLE(STOP_RING)); - xe_hw_engine_mmio_read32(hwe, RING_MI_MODE(0)); + xe_mmio_write32(&hwe->gt->mmio, RING_HWS_PGA(hwe->mmio_base), + xe_bo_ggtt_addr(hwe->hwsp)); } static bool xe_hw_engine_match_fixed_cslice_mode(const struct xe_device *xe, @@ -374,8 +337,8 @@ static bool xe_rtp_cfeg_wmtp_disabled(const struct xe_device *xe, return xe_mmio_read32(&hwe->gt->mmio, XEHP_FUSE4) & CFEG_WMTP_DISABLE; } -void -xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) +static void +hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) { struct xe_gt *gt = hwe->gt; const u8 mocs_write_idx = gt->mocs.uc_index; @@ -390,7 +353,7 @@ xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) * BLIT_CCTL registers are needed to be programmed to un-cached. */ { XE_RTP_NAME("BLIT_CCTL_default_MOCS"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274), ENGINE_CLASS(COPY)), XE_RTP_ACTIONS(FIELD_SET(BLIT_CCTL(0), BLIT_CCTL_DST_MOCS_MASK | @@ -412,6 +375,17 @@ xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) &hwe->reg_lrc, true); } +void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + + xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe); + xe_wa_process_lrc(hwe); + hw_engine_setup_default_lrc_state(hwe); + xe_tuning_process_lrc(hwe); +} + static void hw_engine_setup_default_state(struct xe_hw_engine *hwe) { @@ -436,13 +410,23 @@ hw_engine_setup_default_state(struct xe_hw_engine *hwe) struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); const struct xe_rtp_entry_sr engine_entries[] = { { XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED)), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), XE_RTP_ACTIONS(FIELD_SET(RING_CMD_CCTL(0), CMD_CCTL_WRITE_OVERRIDE_MASK | CMD_CCTL_READ_OVERRIDE_MASK, ring_cmd_cctl_val, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, + { XE_RTP_NAME("Disable HW status page updates for interrupts"), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), + XE_RTP_ACTIONS(SET(RING_HWSTAM(0), ~0x0, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, + { XE_RTP_NAME("Disable engine 'legacy' mode"), + XE_RTP_RULES(FUNC(xe_rtp_match_always)), + XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_DISABLE_LEGACY_MODE, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, /* * To allow the GSC engine to go idle on MTL we need to enable * idle messaging and set the hysteresis value (we use 0xA=5us @@ -465,12 +449,22 @@ hw_engine_setup_default_state(struct xe_hw_engine *hwe) XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), CS_PRIORITY_MEM_READ, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, + { XE_RTP_NAME("Enable CCS Engine(s)"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(RCU_MODE, RCU_MODE_CCS_ENABLE)) + }, /* Use Fixed slice CCS mode */ { XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"), XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)), XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE)) }, + { XE_RTP_NAME("Enable MSI-X interrupt support"), + XE_RTP_RULES(FUNC(xe_rtp_match_has_msix)), + XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_MSIX_INTERRUPT_ENABLE, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, }; xe_rtp_process_to_sr(&ctx, engine_entries, ARRAY_SIZE(engine_entries), @@ -520,9 +514,14 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, hwe->class = info->class; hwe->instance = info->instance; hwe->mmio_base = info->mmio_base; - hwe->irq_offset = xe_device_has_msix(gt_to_xe(gt)) ? - get_msix_irq_offset(gt, info->class) : - info->irq_offset; + if (xe_device_has_msix(gt_to_xe(gt))) { + hwe->irq_offset = get_msix_irq_offset(gt, info->class); + hwe->irq_page = info->instance; + + } else { + hwe->irq_offset = info->irq_offset; + hwe->irq_page = 0; + } hwe->domain = info->domain; hwe->name = info->name; hwe->fence_irq = >->fence_irq[info->class]; @@ -1026,6 +1025,9 @@ bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe) struct xe_gt *gt = hwe->gt; struct xe_device *xe = gt_to_xe(gt); + if (xe_device_is_admin_only(xe)) + return true; + if (hwe->class == XE_ENGINE_CLASS_OTHER) return true; diff --git a/drivers/gpu/drm/xe/xe_hw_engine.h b/drivers/gpu/drm/xe/xe_hw_engine.h index 6b5f9fa2a594..c3ee37f8cfc0 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.h +++ b/drivers/gpu/drm/xe/xe_hw_engine.h @@ -59,7 +59,7 @@ struct xe_hw_engine_snapshot * xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe, struct xe_exec_queue *q); void xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot *snapshot); void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p); -void xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe); +void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe); bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe); @@ -76,7 +76,6 @@ const char *xe_hw_engine_class_to_str(enum xe_engine_class class); u64 xe_hw_engine_read_timestamp(struct xe_hw_engine *hwe); enum xe_force_wake_domains xe_hw_engine_to_fw_domain(struct xe_hw_engine *hwe); -void xe_hw_engine_mmio_write32(struct xe_hw_engine *hwe, struct xe_reg reg, u32 val); u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg); #endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine_types.h b/drivers/gpu/drm/xe/xe_hw_engine_types.h index e4191a7a2c31..2cf898e682f5 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_types.h +++ b/drivers/gpu/drm/xe/xe_hw_engine_types.h @@ -79,7 +79,7 @@ struct xe_hw_engine_class_intf { * @defaults: default scheduling properties */ struct { - /** @sched_props.set_job_timeout: Set job timeout in ms for engine */ + /** @sched_props.job_timeout_ms: Set job timeout in ms for engine */ u32 job_timeout_ms; /** @sched_props.job_timeout_min: Min job timeout in ms for engine */ u32 job_timeout_min; @@ -118,6 +118,8 @@ struct xe_hw_engine { u16 logical_instance; /** @irq_offset: IRQ offset of this hw engine */ u16 irq_offset; + /** @irq_page: MEMIRQ page used by this HW engine */ + u16 irq_page; /** @mmio_base: MMIO base address of this hw engine*/ u32 mmio_base; /** diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c index 2a31b430570e..5135e8e4093f 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.c +++ b/drivers/gpu/drm/xe/xe_hw_error.c @@ -36,11 +36,6 @@ static const char * const hec_uncorrected_fw_errors[] = { "Data Corruption" }; -static const unsigned long xe_hw_error_map[] = { - [XE_GT_ERROR] = DRM_XE_RAS_ERR_COMP_CORE_COMPUTE, - [XE_SOC_ERROR] = DRM_XE_RAS_ERR_COMP_SOC_INTERNAL, -}; - enum gt_vector_regs { ERR_STAT_GT_VECTOR0 = 0, ERR_STAT_GT_VECTOR1, @@ -65,6 +60,18 @@ static enum drm_xe_ras_error_severity hw_err_to_severity(const enum hardware_err return DRM_XE_RAS_ERR_SEV_UNCORRECTABLE; } +static inline u32 err_src_to_id(u32 err_bit) +{ + switch (err_bit) { + case XE_GT_ERROR: + return DRM_XE_RAS_ERR_COMP_CORE_COMPUTE; + case XE_SOC_ERROR: + return DRM_XE_RAS_ERR_COMP_SOC_INTERNAL; + default: + return 0; + } +} + static const char * const pvc_master_global_err_reg[] = { [0 ... 1] = "Undefined", [2] = "HBM SS0: Channel0", @@ -169,11 +176,8 @@ static void csc_hw_error_work(struct work_struct *work) { struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work); struct xe_device *xe = tile_to_xe(tile); - int ret; - ret = xe_survivability_mode_runtime_enable(xe); - if (ret) - drm_err(&xe->drm, "Failed to enable runtime survivability mode\n"); + xe_survivability_mode_runtime_enable(xe); } static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err) @@ -459,14 +463,8 @@ static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_er const char *name; u32 error_id; - /* Check error bit is within bounds */ - if (err_bit >= ARRAY_SIZE(xe_hw_error_map)) - break; - - error_id = xe_hw_error_map[err_bit]; - - /* Check error component is within max */ - if (!error_id || error_id >= DRM_XE_RAS_ERR_COMP_MAX) + error_id = err_src_to_id(err_bit); + if (!error_id) continue; name = info[error_id].name; diff --git a/drivers/gpu/drm/xe/xe_hw_error.h b/drivers/gpu/drm/xe/xe_hw_error.h index d86e28c5180c..5e3a11424108 100644 --- a/drivers/gpu/drm/xe/xe_hw_error.h +++ b/drivers/gpu/drm/xe/xe_hw_error.h @@ -2,8 +2,8 @@ /* * Copyright © 2025 Intel Corporation */ -#ifndef XE_HW_ERROR_H_ -#define XE_HW_ERROR_H_ +#ifndef _XE_HW_ERROR_H_ +#define _XE_HW_ERROR_H_ #include <linux/types.h> diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index 92e423a339f1..de3f2aeffc3f 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -180,6 +180,18 @@ struct xe_hwmon { struct xe_hwmon_thermal_info temp; }; +static inline int prepare_power_limit_param2(const struct xe_hwmon *hwmon) +{ + if (hwmon->boot_power_limit_read) { + if (hwmon->xe->info.platform >= XE_CRESCENTISLAND) + return READ_PL_ACCEPTED; + else + return READ_PL_FROM_PCODE; + } else { + return READ_PL_FROM_FW; + } +} + static int xe_hwmon_pcode_read_power_limit(const struct xe_hwmon *hwmon, u32 attr, int channel, u32 *uval) { @@ -191,9 +203,7 @@ static int xe_hwmon_pcode_read_power_limit(const struct xe_hwmon *hwmon, u32 att (channel == CHANNEL_CARD) ? READ_PSYSGPU_POWER_LIMIT : READ_PACKAGE_POWER_LIMIT, - hwmon->boot_power_limit_read ? - READ_PL_FROM_PCODE : READ_PL_FROM_FW), - &val0, &val1); + prepare_power_limit_param2(hwmon)), &val0, &val1); if (ret) { drm_dbg(&hwmon->xe->drm, "read failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n", @@ -226,10 +236,7 @@ static int xe_hwmon_pcode_rmw_power_limit(const struct xe_hwmon *hwmon, u32 attr (channel == CHANNEL_CARD) ? READ_PSYSGPU_POWER_LIMIT : READ_PACKAGE_POWER_LIMIT, - hwmon->boot_power_limit_read ? - READ_PL_FROM_PCODE : READ_PL_FROM_FW), - &val0, &val1); - + prepare_power_limit_param2(hwmon)), &val0, &val1); if (ret) drm_dbg(&hwmon->xe->drm, "read failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n", channel, val0, val1, ret); @@ -296,7 +303,12 @@ static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg return GT_PERF_STATUS; break; case REG_PKG_ENERGY_STATUS: - if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) { + if (xe->info.platform == XE_CRESCENTISLAND) { + if (channel == CHANNEL_CARD) + return CRI_PLATFORM_ENERGY_STATUS; + else if (channel == CHANNEL_PKG) + return CRI_PACKAGE_ENERGY_STATUS; + } else if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) { return PVC_GT0_PLATFORM_ENERGY_STATUS; } else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG)) { return PCU_CR_PACKAGE_ENERGY_STATUS; diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c index 7560a45f7f64..9e49e2241da4 100644 --- a/drivers/gpu/drm/xe/xe_irq.c +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -24,6 +24,7 @@ #include "xe_mmio.h" #include "xe_pxp.h" #include "xe_sriov.h" +#include "xe_sysctrl.h" #include "xe_tile.h" /* @@ -525,6 +526,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) xe_heci_csc_irq_handler(xe, master_ctl); xe_display_irq_handler(xe, master_ctl); xe_i2c_irq_handler(xe, master_ctl); + xe_sysctrl_irq_handler(xe, master_ctl); xe_mert_irq_handler(xe, master_ctl); gu_misc_iir = gu_misc_irq_ack(xe, master_ctl); } diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 9d12a0d2f0b5..a4292a11391d 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -21,8 +21,10 @@ #include "xe_configfs.h" #include "xe_device.h" #include "xe_drm_client.h" +#include "xe_exec_queue.h" #include "xe_exec_queue_types.h" #include "xe_gt.h" +#include "xe_gt_clock.h" #include "xe_gt_printk.h" #include "xe_hw_fence.h" #include "xe_map.h" @@ -682,25 +684,6 @@ static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe) } } -static int lrc_ring_mi_mode(struct xe_hw_engine *hwe) -{ - struct xe_device *xe = gt_to_xe(hwe->gt); - - if (GRAPHICS_VERx100(xe) >= 1250) - return 0x70; - else - return 0x60; -} - -static void reset_stop_ring(u32 *regs, struct xe_hw_engine *hwe) -{ - int x; - - x = lrc_ring_mi_mode(hwe); - regs[x + 1] &= ~STOP_RING; - regs[x + 1] |= STOP_RING << 16; -} - static inline bool xe_lrc_has_indirect_ring_state(struct xe_lrc *lrc) { return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE; @@ -746,9 +729,16 @@ size_t xe_lrc_reg_size(struct xe_device *xe) return 80 * sizeof(u32); } -size_t xe_lrc_skip_size(struct xe_device *xe) +/** + * xe_lrc_engine_state_size() - Get size of the engine state within LRC + * @gt: the &xe_gt struct instance + * @class: Hardware engine class + * + * Returns: Size of the engine state + */ +size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class) { - return LRC_PPHWSP_SIZE + xe_lrc_reg_size(xe); + return xe_gt_lrc_hang_replay_size(gt, class) - xe_lrc_reg_size(gt_to_xe(gt)); } static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc) @@ -788,6 +778,16 @@ static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc) return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32); } +static u32 __xe_lrc_queue_timestamp_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP * sizeof(u32); +} + +static u32 __xe_lrc_queue_timestamp_udw_offset(struct xe_lrc *lrc) +{ + return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP_UDW * sizeof(u32); +} + static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc) { u32 offset = xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - @@ -837,6 +837,8 @@ DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo) DECL_MAP_ADDR_HELPERS(parallel, lrc->bo) DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo) DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo) +DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo) +DECL_MAP_ADDR_HELPERS(queue_timestamp_udw, lrc->bo) #undef DECL_MAP_ADDR_HELPERS @@ -886,6 +888,29 @@ static u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc) } /** + * xe_lrc_queue_timestamp() - Read queue timestamp value + * @lrc: Pointer to the lrc. + * + * Returns: queue timestamp value + */ +static u64 xe_lrc_queue_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + u32 ldw, udw = 0; + + xe_assert(xe, xe_lrc_is_multi_queue(lrc)); + + map = __xe_lrc_queue_timestamp_map(lrc); + ldw = xe_map_read32(xe, &map); + + map = __xe_lrc_queue_timestamp_udw_map(lrc); + udw = xe_map_read32(xe, &map); + + return (u64)udw << 32 | ldw; +} + +/** * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address * @lrc: Pointer to the lrc. * @@ -980,7 +1005,6 @@ static void *empty_lrc_data(struct xe_hw_engine *hwe) set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe); set_context_control(regs, hwe); set_memory_based_intr(regs, hwe); - reset_stop_ring(regs, hwe); if (xe_gt_has_indirect_ring_state(gt)) { regs = data + xe_gt_lrc_size(gt, hwe->class) - LRC_INDIRECT_RING_STATE_SIZE; @@ -1214,7 +1238,7 @@ static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, if (xe_gt_WARN_ON(lrc->gt, max_len < 3)) return -ENOSPC; - *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_LRM_CS_MMIO | MI_LRI_NUM_REGS(1); *cmd++ = CS_DEBUG_MODE2(0).addr; *cmd++ = REG_MASKED_FIELD_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); @@ -1550,6 +1574,18 @@ static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct if (lrc_to_xe(lrc)->info.has_64bit_timestamp) xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0); + /* + * Note: It's possible that this LRC may belong to an exec_queue that is + * not part of a multi-queue group. That said, it doesn't hurt to set + * this field anyways since any class that supports multi-queue will + * have these LRC fields defined. + */ + if (xe_gt_supports_multi_queue(gt, hwe->class)) { + lrc->queue_timestamp = 0; + xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP, 0); + xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP_UDW, 0); + } + if (xe->info.has_asid && vm) xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid); @@ -2475,7 +2511,17 @@ struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) snapshot->replay_offset = 0; snapshot->replay_size = lrc->replay_size; snapshot->lrc_snapshot = NULL; - snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc)); + snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); + snapshot->ctx_timestamp_ms = + xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc)); + if (xe_lrc_is_multi_queue(lrc)) { + snapshot->queue_timestamp = xe_lrc_queue_timestamp(lrc); + snapshot->queue_timestamp_ms = + xe_gt_clock_interval_to_ms(lrc->gt, snapshot->queue_timestamp); + } else { + snapshot->queue_timestamp = 0; + snapshot->queue_timestamp_ms = 0; + } snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); return snapshot; } @@ -2528,7 +2574,10 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); - drm_printf(p, "\tTimestamp: 0x%08x\n", snapshot->ctx_timestamp); + drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp); + drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms); + drm_printf(p, "\tQueue Timestamp: 0x%016llx\n", snapshot->queue_timestamp); + drm_printf(p, "\tQueue Timestamp ms: %llu\n", snapshot->queue_timestamp_ms); drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); if (!snapshot->lrc_snapshot) @@ -2569,17 +2618,27 @@ void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot) kfree(snapshot); } -static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) +static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id) { u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id); u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id); + struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false); + + if (xe_gt_WARN_ONCE(gt, !hwe || xe_hw_engine_is_reserved(hwe), + "Unexpected engine class:instance %d:%d for utilization\n", + class, instance)) + return NULL; + + return hwe; +} + +static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) +{ struct xe_hw_engine *hwe; u64 val; - hwe = xe_gt_hw_engine(lrc->gt, class, instance, false); - if (xe_gt_WARN_ONCE(lrc->gt, !hwe || xe_hw_engine_is_reserved(hwe), - "Unexpected engine class:instance %d:%d for context utilization\n", - class, instance)) + hwe = engine_id_to_hwe(lrc->gt, engine_id); + if (!hwe) return -1; if (lrc_to_xe(lrc)->info.has_64bit_timestamp) @@ -2594,66 +2653,136 @@ static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) return 0; } -/** - * xe_lrc_timestamp() - Current ctx timestamp - * @lrc: Pointer to the lrc. - * - * Return latest ctx timestamp. With support for active contexts, the - * calculation may be slightly racy, so follow a read-again logic to ensure that - * the context is still active before returning the right timestamp. - * - * Returns: New ctx timestamp value - */ -u64 xe_lrc_timestamp(struct xe_lrc *lrc) +static u64 get_queue_timestamp(struct xe_hw_engine *hwe) { - u64 lrc_ts, reg_ts, new_ts = lrc->ctx_timestamp; - u32 engine_id; + return xe_mmio_read64_2x32(&hwe->gt->mmio, + RING_QUEUE_TIMESTAMP(hwe->mmio_base)); +} - lrc_ts = xe_lrc_ctx_timestamp(lrc); - /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ - if (IS_SRIOV_VF(lrc_to_xe(lrc))) { - new_ts = lrc_ts; - goto done; - } +static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe) +{ + u32 val = xe_mmio_read32(&hwe->gt->mmio, + RING_CSMQDEBUG(hwe->mmio_base)); - if (lrc_ts == CONTEXT_ACTIVE) { - engine_id = xe_lrc_engine_id(lrc); - if (!get_ctx_timestamp(lrc, engine_id, ®_ts)) - new_ts = reg_ts; + return REG_FIELD_GET(CURRENT_ACTIVE_QUEUE_ID_MASK, val); +} - /* read lrc again to ensure context is still active */ - lrc_ts = xe_lrc_ctx_timestamp(lrc); - } +static bool context_active(struct xe_lrc *lrc) +{ + return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE; +} + +static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct xe_lrc *primary_lrc = lrc->multi_queue.primary_lrc; + struct xe_hw_engine *hwe; + u64 reg_queue_ts = lrc->queue_timestamp; + + if (IS_SRIOV_VF(xe)) + return xe_lrc_queue_timestamp(lrc); + + xe_assert(xe, primary_lrc); + + /* WA BB populates CONTEXT_ACTIVE cookie for primary context only */ + if (!context_active(primary_lrc)) + return xe_lrc_queue_timestamp(lrc); + + /* WA BB populates engine id in PPHWSP of primary context only */ + hwe = engine_id_to_hwe(primary_lrc->gt, xe_lrc_engine_id(primary_lrc)); + if (!hwe) + return xe_lrc_queue_timestamp(lrc); + + if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos) + return xe_lrc_queue_timestamp(lrc); + + /* queue is active, so store the queue timestamp register */ + reg_queue_ts = get_queue_timestamp(hwe); + + /* double check queue and primary queue are both still active */ + if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos || + !context_active(primary_lrc)) + return xe_lrc_queue_timestamp(lrc); + + return reg_queue_ts; +} + +static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + *old_ts = lrc->queue_timestamp; + lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc); + + trace_xe_lrc_update_queue_timestamp(lrc, *old_ts); + + return lrc->queue_timestamp; +} + +static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc) +{ + u64 reg_ts, new_ts = lrc->ctx_timestamp; + + /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ + if (IS_SRIOV_VF(lrc_to_xe(lrc))) + return xe_lrc_ctx_timestamp(lrc); + + if (context_active(lrc) && + !get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts)) + new_ts = reg_ts; /* - * If context switched out, just use the lrc_ts. Note that this needs to - * be a separate if condition. + * If context swicthed out while we were here, just return the latest + * LRC CTX TIMESTAMP value. */ - if (lrc_ts != CONTEXT_ACTIVE) - new_ts = lrc_ts; + if (!context_active(lrc)) + return xe_lrc_ctx_timestamp(lrc); -done: return new_ts; } +static u64 xe_lrc_update_context_timestamp(struct xe_lrc *lrc, u64 *old_ts) +{ + *old_ts = lrc->ctx_timestamp; + lrc->ctx_timestamp = xe_lrc_context_timestamp(lrc); + + trace_xe_lrc_update_timestamp(lrc, *old_ts); + + return lrc->ctx_timestamp; +} + /** - * xe_lrc_update_timestamp() - Update ctx timestamp + * xe_lrc_timestamp() - Current lrc timestamp + * @lrc: Pointer to the lrc. + * + * Return latest lrc timestamp. With support for active contexts/queues, the + * calculation may be slightly racy, so follow a read-again logic to ensure that + * the context/queue is still active before returning the right timestamp. + * + * Returns: New lrc timestamp value + */ +u64 xe_lrc_timestamp(struct xe_lrc *lrc) +{ + if (xe_lrc_is_multi_queue(lrc)) + return xe_lrc_multi_queue_timestamp(lrc); + else + return xe_lrc_context_timestamp(lrc); +} + +/** + * xe_lrc_update_timestamp() - Update lrc timestamp * @lrc: Pointer to the lrc. * @old_ts: Old timestamp value * - * Populate @old_ts current saved ctx timestamp, read new ctx timestamp and + * Populate @old_ts with current saved lrc timestamp, read new lrc timestamp and * update saved value. * - * Returns: New ctx timestamp value + * Returns: New lrc timestamp value */ u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts) { - *old_ts = lrc->ctx_timestamp; - lrc->ctx_timestamp = xe_lrc_timestamp(lrc); - - trace_xe_lrc_update_timestamp(lrc, *old_ts); - - return lrc->ctx_timestamp; + if (xe_lrc_is_multi_queue(lrc)) + return xe_lrc_update_multi_queue_timestamp(lrc, old_ts); + else + return xe_lrc_update_context_timestamp(lrc, old_ts); } /** diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index e7c975f9e2d9..0a3a611391ee 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -37,7 +37,10 @@ struct xe_lrc_snapshot { } tail; u32 start_seqno; u32 seqno; - u32 ctx_timestamp; + u64 ctx_timestamp; + u64 ctx_timestamp_ms; + u64 queue_timestamp; + u64 queue_timestamp_ms; u32 ctx_job_timestamp; }; @@ -90,6 +93,11 @@ static inline size_t xe_lrc_ring_size(void) return SZ_16K; } +static inline bool xe_lrc_is_multi_queue(struct xe_lrc *lrc) +{ + return lrc->multi_queue.primary_lrc; +} + size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class); size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class); u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc); @@ -130,7 +138,7 @@ u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc); struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc); size_t xe_lrc_reg_size(struct xe_device *xe); -size_t xe_lrc_skip_size(struct xe_device *xe); +size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class); void xe_lrc_dump_default(struct drm_printer *p, struct xe_gt *gt, diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h index 5a718f759ed6..53ef48feebfc 100644 --- a/drivers/gpu/drm/xe/xe_lrc_types.h +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -63,6 +63,17 @@ struct xe_lrc { /** @ctx_timestamp: readout value of CTX_TIMESTAMP on last update */ u64 ctx_timestamp; + + /** @queue_timestamp: value of QUEUE_TIMESTAMP on last update */ + u64 queue_timestamp; + + /** @multi_queue: Multi queue LRC related information */ + struct { + /** @multi_queue.primary_lrc: Primary lrc of this multi-queue group*/ + struct xe_lrc *primary_lrc; + /** @multi_queue.pos: Position of LRC within the multi-queue group */ + u8 pos; + } multi_queue; }; struct xe_lrc_snapshot; diff --git a/drivers/gpu/drm/xe/xe_mem_pool.c b/drivers/gpu/drm/xe/xe_mem_pool.c new file mode 100644 index 000000000000..d5e24d6aa88d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mem_pool.c @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include <linux/kernel.h> + +#include <drm/drm_managed.h> + +#include "instructions/xe_mi_commands.h" +#include "xe_bo.h" +#include "xe_device_types.h" +#include "xe_map.h" +#include "xe_mem_pool.h" +#include "xe_mem_pool_types.h" +#include "xe_tile_printk.h" + +/** + * struct xe_mem_pool - DRM MM pool for sub-allocating memory from a BO on an + * XE tile. + * + * The XE memory pool is a DRM MM manager that provides sub-allocation of memory + * from a backing buffer object (BO) on a specific XE tile. It is designed to + * manage memory for GPU workloads, allowing for efficient allocation and + * deallocation of memory regions within the BO. + * + * The memory pool maintains a primary BO that is pinned in the GGTT and mapped + * into the CPU address space for direct access. Optionally, it can also maintain + * a shadow BO that can be used for atomic updates to the primary BO's contents. + * + * The API provided by the memory pool allows clients to allocate and free memory + * regions, retrieve GPU and CPU addresses, and synchronize data between the + * primary and shadow BOs as needed. + */ +struct xe_mem_pool { + /** @base: Range allocator over [0, @size) in bytes */ + struct drm_mm base; + /** @bo: Active pool BO (GGTT-pinned, CPU-mapped). */ + struct xe_bo *bo; + /** @shadow: Shadow BO for atomic command updates. */ + struct xe_bo *shadow; + /** @swap_guard: Timeline guard updating @bo and @shadow */ + struct mutex swap_guard; + /** @cpu_addr: CPU virtual address of the active BO. */ + void *cpu_addr; + /** @is_iomem: Indicates if the BO mapping is I/O memory. */ + bool is_iomem; +}; + +static struct xe_mem_pool *node_to_pool(struct xe_mem_pool_node *node) +{ + return container_of(node->sa_node.mm, struct xe_mem_pool, base); +} + +static struct xe_tile *pool_to_tile(struct xe_mem_pool *pool) +{ + return pool->bo->tile; +} + +static void fini_pool_action(struct drm_device *drm, void *arg) +{ + struct xe_mem_pool *pool = arg; + + if (pool->is_iomem) + kvfree(pool->cpu_addr); + + drm_mm_takedown(&pool->base); +} + +static int pool_shadow_init(struct xe_mem_pool *pool) +{ + struct xe_tile *tile = pool->bo->tile; + struct xe_device *xe = tile_to_xe(tile); + struct xe_bo *shadow; + int ret; + + xe_assert(xe, !pool->shadow); + + ret = drmm_mutex_init(&xe->drm, &pool->swap_guard); + if (ret) + return ret; + + if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&pool->swap_guard); + fs_reclaim_release(GFP_KERNEL); + } + shadow = xe_managed_bo_create_pin_map(xe, tile, + xe_bo_size(pool->bo), + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(shadow)) + return PTR_ERR(shadow); + + pool->shadow = shadow; + + return 0; +} + +/** + * xe_mem_pool_init() - Initialize memory pool. + * @tile: the &xe_tile where allocate. + * @size: number of bytes to allocate. + * @guard: the size of the guard region at the end of the BO that is not + * sub-allocated, in bytes. + * @flags: flags to use to create shadow pool. + * + * Initializes a memory pool for sub-allocating memory from a backing BO on the + * specified XE tile. The backing BO is pinned in the GGTT and mapped into + * the CPU address space for direct access. Optionally, a shadow BO can also be + * initialized for atomic updates to the primary BO's contents. + * + * Returns: a pointer to the &xe_mem_pool, or an error pointer on failure. + */ +struct xe_mem_pool *xe_mem_pool_init(struct xe_tile *tile, u32 size, + u32 guard, int flags) +{ + struct xe_device *xe = tile_to_xe(tile); + struct xe_mem_pool *pool; + struct xe_bo *bo; + u32 managed_size; + int ret; + + xe_tile_assert(tile, size > guard); + managed_size = size - guard; + + pool = drmm_kzalloc(&xe->drm, sizeof(*pool), GFP_KERNEL); + if (!pool) + return ERR_PTR(-ENOMEM); + + bo = xe_managed_bo_create_pin_map(xe, tile, size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(bo)) { + xe_tile_err(tile, "Failed to prepare %uKiB BO for mem pool (%pe)\n", + size / SZ_1K, bo); + return ERR_CAST(bo); + } + pool->bo = bo; + pool->is_iomem = bo->vmap.is_iomem; + + if (pool->is_iomem) { + pool->cpu_addr = kvzalloc(size, GFP_KERNEL); + if (!pool->cpu_addr) + return ERR_PTR(-ENOMEM); + } else { + pool->cpu_addr = bo->vmap.vaddr; + } + + if (flags & XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY) { + ret = pool_shadow_init(pool); + + if (ret) + goto out_err; + } + + drm_mm_init(&pool->base, 0, managed_size); + ret = drmm_add_action_or_reset(&xe->drm, fini_pool_action, pool); + if (ret) + return ERR_PTR(ret); + + return pool; + +out_err: + if (flags & XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY) + xe_tile_err(tile, + "Failed to initialize shadow BO for mem pool (%d)\n", ret); + if (bo->vmap.is_iomem) + kvfree(pool->cpu_addr); + return ERR_PTR(ret); +} + +/** + * xe_mem_pool_sync() - Copy the entire contents of the main pool to shadow pool. + * @pool: the memory pool containing the primary and shadow BOs. + * + * Copies the entire contents of the primary pool to the shadow pool. This must + * be done after xe_mem_pool_init() with the XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY + * flag to ensure that the shadow pool has the same initial contents as the primary + * pool. After this initial synchronization, clients can choose to synchronize the + * shadow pool with the primary pool on a node basis using + * xe_mem_pool_sync_shadow_locked() as needed. + * + * Return: None. + */ +void xe_mem_pool_sync(struct xe_mem_pool *pool) +{ + struct xe_tile *tile = pool_to_tile(pool); + struct xe_device *xe = tile_to_xe(tile); + + xe_tile_assert(tile, pool->shadow); + + xe_map_memcpy_to(xe, &pool->shadow->vmap, 0, + pool->cpu_addr, xe_bo_size(pool->bo)); +} + +/** + * xe_mem_pool_swap_shadow_locked() - Swap the primary BO with the shadow BO. + * @pool: the memory pool containing the primary and shadow BOs. + * + * Swaps the primary buffer object with the shadow buffer object in the mem + * pool. This allows for atomic updates to the contents of the primary BO + * by first writing to the shadow BO and then swapping it with the primary BO. + * Swap_guard must be held to ensure synchronization with any concurrent swap + * operations. + * + * Return: None. + */ +void xe_mem_pool_swap_shadow_locked(struct xe_mem_pool *pool) +{ + struct xe_tile *tile = pool_to_tile(pool); + + xe_tile_assert(tile, pool->shadow); + lockdep_assert_held(&pool->swap_guard); + + swap(pool->bo, pool->shadow); + if (!pool->bo->vmap.is_iomem) + pool->cpu_addr = pool->bo->vmap.vaddr; +} + +/** + * xe_mem_pool_sync_shadow_locked() - Copy node from primary pool to shadow pool. + * @node: the node allocated in the memory pool. + * + * Copies the specified batch buffer from the primary pool to the shadow pool. + * Swap_guard must be held to ensure synchronization with any concurrent swap + * operations. + * + * Return: None. + */ +void xe_mem_pool_sync_shadow_locked(struct xe_mem_pool_node *node) +{ + struct xe_mem_pool *pool = node_to_pool(node); + struct xe_tile *tile = pool_to_tile(pool); + struct xe_device *xe = tile_to_xe(tile); + struct drm_mm_node *sa_node = &node->sa_node; + + xe_tile_assert(tile, pool->shadow); + lockdep_assert_held(&pool->swap_guard); + + xe_map_memcpy_to(xe, &pool->shadow->vmap, + sa_node->start, + pool->cpu_addr + sa_node->start, + sa_node->size); +} + +/** + * xe_mem_pool_gpu_addr() - Retrieve GPU address of memory pool. + * @pool: the memory pool + * + * Returns: GGTT address of the memory pool. + */ +u64 xe_mem_pool_gpu_addr(struct xe_mem_pool *pool) +{ + return xe_bo_ggtt_addr(pool->bo); +} + +/** + * xe_mem_pool_cpu_addr() - Retrieve CPU address of manager pool. + * @pool: the memory pool + * + * Returns: CPU virtual address of memory pool. + */ +void *xe_mem_pool_cpu_addr(struct xe_mem_pool *pool) +{ + return pool->cpu_addr; +} + +/** + * xe_mem_pool_bo_swap_guard() - Retrieve the mutex used to guard swap + * operations on a memory pool. + * @pool: the memory pool + * + * Returns: Swap guard mutex or NULL if shadow pool is not created. + */ +struct mutex *xe_mem_pool_bo_swap_guard(struct xe_mem_pool *pool) +{ + if (!pool->shadow) + return NULL; + + return &pool->swap_guard; +} + +/** + * xe_mem_pool_bo_flush_write() - Copy the data from the sub-allocation + * to the GPU memory. + * @node: the node allocated in the memory pool to flush. + */ +void xe_mem_pool_bo_flush_write(struct xe_mem_pool_node *node) +{ + struct xe_mem_pool *pool = node_to_pool(node); + struct xe_tile *tile = pool_to_tile(pool); + struct xe_device *xe = tile_to_xe(tile); + struct drm_mm_node *sa_node = &node->sa_node; + + if (!pool->bo->vmap.is_iomem) + return; + + xe_map_memcpy_to(xe, &pool->bo->vmap, sa_node->start, + pool->cpu_addr + sa_node->start, + sa_node->size); +} + +/** + * xe_mem_pool_bo_sync_read() - Copy the data from GPU memory to the + * sub-allocation. + * @node: the node allocated in the memory pool to read back. + */ +void xe_mem_pool_bo_sync_read(struct xe_mem_pool_node *node) +{ + struct xe_mem_pool *pool = node_to_pool(node); + struct xe_tile *tile = pool_to_tile(pool); + struct xe_device *xe = tile_to_xe(tile); + struct drm_mm_node *sa_node = &node->sa_node; + + if (!pool->bo->vmap.is_iomem) + return; + + xe_map_memcpy_from(xe, pool->cpu_addr + sa_node->start, + &pool->bo->vmap, sa_node->start, sa_node->size); +} + +/** + * xe_mem_pool_alloc_node() - Allocate a new node for use with xe_mem_pool. + * + * Returns: node structure or an ERR_PTR(-ENOMEM). + */ +struct xe_mem_pool_node *xe_mem_pool_alloc_node(void) +{ + struct xe_mem_pool_node *node = kzalloc_obj(*node); + + if (!node) + return ERR_PTR(-ENOMEM); + + return node; +} + +/** + * xe_mem_pool_insert_node() - Insert a node into the memory pool. + * @pool: the memory pool to insert into + * @node: the node to insert + * @size: the size of the node to be allocated in bytes. + * + * Inserts a node into the specified memory pool using drm_mm for + * allocation. + * + * Returns: 0 on success or a negative error code on failure. + */ +int xe_mem_pool_insert_node(struct xe_mem_pool *pool, + struct xe_mem_pool_node *node, u32 size) +{ + if (!pool) + return -EINVAL; + + return drm_mm_insert_node(&pool->base, &node->sa_node, size); +} + +/** + * xe_mem_pool_free_node() - Free a node allocated from the memory pool. + * @node: the node to free + * + * Returns: None. + */ +void xe_mem_pool_free_node(struct xe_mem_pool_node *node) +{ + if (!node) + return; + + drm_mm_remove_node(&node->sa_node); + kfree(node); +} + +/** + * xe_mem_pool_node_cpu_addr() - Retrieve CPU address of the node. + * @node: the node allocated in the memory pool + * + * Returns: CPU virtual address of the node. + */ +void *xe_mem_pool_node_cpu_addr(struct xe_mem_pool_node *node) +{ + struct xe_mem_pool *pool = node_to_pool(node); + + return xe_mem_pool_cpu_addr(pool) + node->sa_node.start; +} + +/** + * xe_mem_pool_dump() - Dump the state of the DRM MM manager for debugging. + * @pool: the memory pool info be dumped. + * @p: The DRM printer to use for output. + * + * Only the drm managed region is dumped, not the state of the BOs or any other + * pool information. + * + * Returns: None. + */ +void xe_mem_pool_dump(struct xe_mem_pool *pool, struct drm_printer *p) +{ + drm_mm_print(&pool->base, p); +} diff --git a/drivers/gpu/drm/xe/xe_mem_pool.h b/drivers/gpu/drm/xe/xe_mem_pool.h new file mode 100644 index 000000000000..89cd2555fe91 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mem_pool.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ +#ifndef _XE_MEM_POOL_H_ +#define _XE_MEM_POOL_H_ + +#include <linux/sizes.h> +#include <linux/types.h> + +#include <drm/drm_mm.h> +#include "xe_mem_pool_types.h" + +struct drm_printer; +struct xe_mem_pool; +struct xe_tile; + +struct xe_mem_pool *xe_mem_pool_init(struct xe_tile *tile, u32 size, + u32 guard, int flags); +void xe_mem_pool_sync(struct xe_mem_pool *pool); +void xe_mem_pool_swap_shadow_locked(struct xe_mem_pool *pool); +void xe_mem_pool_sync_shadow_locked(struct xe_mem_pool_node *node); +u64 xe_mem_pool_gpu_addr(struct xe_mem_pool *pool); +void *xe_mem_pool_cpu_addr(struct xe_mem_pool *pool); +struct mutex *xe_mem_pool_bo_swap_guard(struct xe_mem_pool *pool); +void xe_mem_pool_bo_flush_write(struct xe_mem_pool_node *node); +void xe_mem_pool_bo_sync_read(struct xe_mem_pool_node *node); +struct xe_mem_pool_node *xe_mem_pool_alloc_node(void); +int xe_mem_pool_insert_node(struct xe_mem_pool *pool, + struct xe_mem_pool_node *node, u32 size); +void xe_mem_pool_free_node(struct xe_mem_pool_node *node); +void *xe_mem_pool_node_cpu_addr(struct xe_mem_pool_node *node); +void xe_mem_pool_dump(struct xe_mem_pool *pool, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_mem_pool_types.h b/drivers/gpu/drm/xe/xe_mem_pool_types.h new file mode 100644 index 000000000000..d5e926c93351 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mem_pool_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_MEM_POOL_TYPES_H_ +#define _XE_MEM_POOL_TYPES_H_ + +#include <drm/drm_mm.h> + +#define XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY BIT(0) + +/** + * struct xe_mem_pool_node - Sub-range allocations from mem pool. + */ +struct xe_mem_pool_node { + /** @sa_node: drm_mm_node for this allocation. */ + struct drm_mm_node sa_node; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c index 811e07136efb..9dfe965cb46e 100644 --- a/drivers/gpu/drm/xe/xe_memirq.c +++ b/drivers/gpu/drm/xe/xe_memirq.c @@ -152,6 +152,16 @@ static const char *guc_name(struct xe_guc *guc) * */ +/* ISR */ +#define XE_MEMIRQ_STATUS_OFFSET(inst) ((inst) * SZ_4K + 0x0) +/* IIR */ +#define XE_MEMIRQ_SOURCE_OFFSET(inst) ((inst) * SZ_4K + 0x400) +/* IMR */ +#define XE_MEMIRQ_ENABLE_OFFSET 0x440 + +/* engine ISR vector offset */ +#define XE_MEMIRQ_VECTOR_OFFSET(page, source) (XE_MEMIRQ_STATUS_OFFSET(page) + (source) * SZ_16) + static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq) { /* @@ -162,18 +172,35 @@ static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq) return xe_device_has_msix(memirq_to_xe(memirq)); } +static unsigned int hwe_max_count(struct xe_tile *tile) +{ + unsigned int max_instance = 0; + unsigned int gtid, hweid; + struct xe_hw_engine *hwe; + struct xe_gt *gt; + + for_each_gt_on_tile(gt, tile, gtid) + for_each_hw_engine(hwe, gt, hweid) + max_instance = max(max_instance, hwe->instance); + + return max_instance + 1; +} + static int memirq_alloc_pages(struct xe_memirq *memirq) { struct xe_device *xe = memirq_to_xe(memirq); struct xe_tile *tile = memirq_to_tile(memirq); - size_t bo_size = hw_reports_to_instance_zero(memirq) ? - XE_HW_ENGINE_MAX_INSTANCE * SZ_4K : SZ_4K; + unsigned int num_pages; struct xe_bo *bo; + size_t bo_size; int err; BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET(0), SZ_64)); BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET(0), SZ_4K)); + num_pages = hw_reports_to_instance_zero(memirq) ? hwe_max_count(tile) : 1; + bo_size = num_pages * SZ_4K; + bo = xe_managed_bo_create_pin_map(xe, tile, bo_size, XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT | @@ -192,16 +219,15 @@ static int memirq_alloc_pages(struct xe_memirq *memirq) memirq->bo = bo; memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET(0)); - memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET(0)); - memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET); + memirq->num_pages = num_pages; memirq_assert(memirq, !memirq->source.is_iomem); - memirq_assert(memirq, !memirq->status.is_iomem); - memirq_assert(memirq, !memirq->mask.is_iomem); - memirq_debug(memirq, "page offsets: bo %#x bo_size %zu source %#x status %#x\n", - xe_bo_ggtt_addr(bo), bo_size, XE_MEMIRQ_SOURCE_OFFSET(0), - XE_MEMIRQ_STATUS_OFFSET(0)); + memirq_debug(memirq, "pages: count %u size %zu\n", num_pages, bo_size); + memirq_debug(memirq, "page0: source %#x status %#x mask %#x\n", + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_SOURCE_OFFSET(0), + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_STATUS_OFFSET(0), + xe_bo_ggtt_addr(bo) + XE_MEMIRQ_ENABLE_OFFSET); return 0; @@ -212,7 +238,12 @@ out: static void memirq_set_enable(struct xe_memirq *memirq, bool enable) { - iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0); + /* + * We only care about the GT_MI_USER_INTERRUPT from the engines and + * the GuC does not look at the ENABLE mask at all. + */ + iosys_map_wr(&memirq->bo->vmap, XE_MEMIRQ_ENABLE_OFFSET, u32, + enable ? GT_MI_USER_INTERRUPT : 0); memirq->enabled = enable; } @@ -249,15 +280,6 @@ int xe_memirq_init(struct xe_memirq *memirq) return 0; } -static u32 __memirq_source_page(struct xe_memirq *memirq, u16 instance) -{ - memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE); - memirq_assert(memirq, memirq->bo); - - instance = hw_reports_to_instance_zero(memirq) ? instance : 0; - return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(instance); -} - /** * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_. * @memirq: the &xe_memirq to query @@ -272,16 +294,7 @@ u32 xe_memirq_source_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe) { memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); - return __memirq_source_page(memirq, hwe->instance); -} - -static u32 __memirq_status_page(struct xe_memirq *memirq, u16 instance) -{ - memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE); - memirq_assert(memirq, memirq->bo); - - instance = hw_reports_to_instance_zero(memirq) ? instance : 0; - return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(instance); + return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(hwe->irq_page); } /** @@ -298,7 +311,7 @@ u32 xe_memirq_status_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe) { memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); - return __memirq_status_page(memirq, hwe->instance); + return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(hwe->irq_page); } /** @@ -335,13 +348,14 @@ int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc) { bool is_media = xe_gt_is_media_type(guc_to_gt(guc)); u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC); - u32 source, status; + u64 source, status; int err; memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq))); - source = __memirq_source_page(memirq, 0) + offset; - status = __memirq_status_page(memirq, 0) + offset * SZ_16; + /* GuC expects exact locations, it doesn't add anything on its own */ + source = xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(0) + offset; + status = xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_VECTOR_OFFSET(0, offset); err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY, source); @@ -427,13 +441,25 @@ static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector, return __memirq_received(memirq, vector, offset, name, true); } +static void memirq_assume_received(struct xe_memirq *memirq, const char *source, + u16 offset, const char *status) +{ + memirq_debug(memirq, "ASSUME %s %s(%u)\n", source, status, offset); +} + static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status, struct xe_hw_engine *hwe) { memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr); - if (memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name)) - xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT); + /* + * The programming note says to assume that GT_MI_USER_INTERRUPT is always + * set. Check and clear related status byte just for a debug. + */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) && + !memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name)) + memirq_assume_received(memirq, hwe->name, ilog2(GT_MI_USER_INTERRUPT), "USER"); + xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT); } static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status, @@ -443,8 +469,14 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr); - if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name)) - xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST); + /* + * The programming note says to assume that GUC_INTR_GUC2HOST is always + * set. Check and clear related status byte just for a debug. + */ + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) && + !memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name)) + memirq_assume_received(memirq, name, ilog2(GUC_INTR_GUC2HOST), "GUC2HOST"); + xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST); /* * This is a software interrupt that must be cleared after it's consumed @@ -467,16 +499,16 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat */ void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe) { - u16 offset = hwe->irq_offset; - u16 instance = hw_reports_to_instance_zero(memirq) ? hwe->instance : 0; - struct iosys_map src_offset = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, - XE_MEMIRQ_SOURCE_OFFSET(instance)); + struct iosys_map source = + IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_SOURCE_OFFSET(hwe->irq_page)); - if (memirq_received(memirq, &src_offset, offset, "SRC")) { - struct iosys_map status_offset = + if (memirq_received(memirq, &source, hwe->irq_offset, "SRC")) { + struct iosys_map status = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, - XE_MEMIRQ_STATUS_OFFSET(instance) + offset * SZ_16); - memirq_dispatch_engine(memirq, &status_offset, hwe); + XE_MEMIRQ_VECTOR_OFFSET(hwe->irq_page, + hwe->irq_offset)); + memirq_dispatch_engine(memirq, &status, hwe); } } @@ -491,12 +523,25 @@ bool xe_memirq_guc_sw_int_0_irq_pending(struct xe_memirq *memirq, struct xe_guc { struct xe_gt *gt = guc_to_gt(guc); u32 offset = xe_gt_is_media_type(gt) ? ilog2(INTR_MGUC) : ilog2(INTR_GUC); - struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&memirq->status, offset * SZ_16); + struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, offset)); return memirq_received_noclear(memirq, &map, ilog2(GUC_INTR_SW_INT_0), guc_name(guc)); } +static void memirq_dump_source_pages(struct xe_memirq *memirq) +{ + memirq_assert(memirq, !memirq->bo->vmap.is_iomem); + + for (int n = 0; n < memirq->num_pages; n++) { + memirq_debug(memirq, "SOURCE %*ph\n", 32, + memirq->bo->vmap.vaddr + XE_MEMIRQ_SOURCE_OFFSET(n)); + memirq_debug(memirq, "SOURCE %*ph\n", 32, + memirq->bo->vmap.vaddr + XE_MEMIRQ_SOURCE_OFFSET(n) + 32); + } +} + /** * xe_memirq_handler - The `Memory Based Interrupts`_ Handler. * @memirq: the &xe_memirq @@ -516,9 +561,8 @@ void xe_memirq_handler(struct xe_memirq *memirq) if (!memirq->bo) return; - memirq_assert(memirq, !memirq->source.is_iomem); - memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr); - memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32); + if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ)) + memirq_dump_source_pages(memirq); for_each_gt(gt, xe, gtid) { if (gt->tile != tile) @@ -531,7 +575,8 @@ void xe_memirq_handler(struct xe_memirq *memirq) /* GuC and media GuC (if present) must be checked separately */ if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) { - map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16); + map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, ilog2(INTR_GUC))); memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc); } @@ -539,7 +584,8 @@ void xe_memirq_handler(struct xe_memirq *memirq) return; if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) { - map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16); + map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap, + XE_MEMIRQ_VECTOR_OFFSET(0, ilog2(INTR_MGUC))); memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc); } } diff --git a/drivers/gpu/drm/xe/xe_memirq_types.h b/drivers/gpu/drm/xe/xe_memirq_types.h index 9d0f6c1cdb9d..c2b0b33d55cd 100644 --- a/drivers/gpu/drm/xe/xe_memirq_types.h +++ b/drivers/gpu/drm/xe/xe_memirq_types.h @@ -10,27 +10,18 @@ struct xe_bo; -/* ISR */ -#define XE_MEMIRQ_STATUS_OFFSET(inst) ((inst) * SZ_4K + 0x0) -/* IIR */ -#define XE_MEMIRQ_SOURCE_OFFSET(inst) ((inst) * SZ_4K + 0x400) -/* IMR */ -#define XE_MEMIRQ_ENABLE_OFFSET 0x440 - /** * struct xe_memirq - Data used by the `Memory Based Interrupts`_. * * @bo: buffer object with `Memory Based Interrupts Page Layout`_. + * @num_pages: number of per-instance source/status pages. * @source: iosys pointer to `Interrupt Source Report Page`_. - * @status: iosys pointer to `Interrupt Status Report Page`_. - * @mask: iosys pointer to Interrupt Enable Mask. * @enabled: internal flag used to control processing of the interrupts. */ struct xe_memirq { struct xe_bo *bo; + unsigned int num_pages; struct iosys_map source; - struct iosys_map status; - struct iosys_map mask; bool enabled; }; diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index fc918b4fba54..9428dd5e7760 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -29,7 +29,9 @@ #include "xe_hw_engine.h" #include "xe_lrc.h" #include "xe_map.h" +#include "xe_mem_pool.h" #include "xe_mocs.h" +#include "xe_pat.h" #include "xe_printk.h" #include "xe_pt.h" #include "xe_res_cursor.h" @@ -216,7 +218,7 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, struct xe_vm *vm, u32 *ofs) { struct xe_device *xe = tile_to_xe(tile); - u16 pat_index = xe->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); u8 id = tile->id; u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; #define VRAM_IDENTITY_MAP_COUNT 2 @@ -336,7 +338,7 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, * if flat ccs is enabled. */ if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) { - u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION]; + u16 comp_pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION); u64 vram_offset = IDENTITY_OFFSET + DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G); u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE; @@ -636,10 +638,10 @@ static void emit_pte(struct xe_migrate *m, /* Indirect access needs compression enabled uncached PAT index */ if (GRAPHICS_VERx100(xe) >= 2000) - pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] : - xe->pat.idx[XE_CACHE_WB]; + pat_index = is_comp_pte ? xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION) : + xe_cache_pat_idx(xe, XE_CACHE_WB); else - pat_index = xe->pat.idx[XE_CACHE_WB]; + pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); @@ -726,7 +728,22 @@ static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, bb->len = cs - bb->cs; } -#define EMIT_COPY_DW 10 +static u32 blt_fast_copy_cmd_len(struct xe_device *xe) +{ + return 10; +} + +static u32 blt_mem_copy_cmd_len(struct xe_device *xe) +{ + return 10; +} + +static u32 emit_copy_cmd_len(struct xe_device *xe) +{ + return (xe->info.has_mem_copy_instr) ? blt_mem_copy_cmd_len(xe) : + blt_fast_copy_cmd_len(xe); +} + static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u64 dst_ofs, unsigned int size, unsigned int pitch) @@ -734,6 +751,7 @@ static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, struct xe_device *xe = gt_to_xe(gt); u32 mocs = 0; u32 tile_y = 0; + u32 len; xe_gt_assert(gt, !(pitch & 3)); xe_gt_assert(gt, size / pitch <= S16_MAX); @@ -746,7 +764,8 @@ static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, if (GRAPHICS_VERx100(xe) >= 1250) tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; - bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); + len = blt_fast_copy_cmd_len(xe); + bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (len - 2); bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; bb->cs[bb->len++] = 0; bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; @@ -763,6 +782,7 @@ static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u64 dst_ofs, unsigned int size, unsigned int pitch) { u32 mode, copy_type, width; + u32 len; xe_gt_assert(gt, IS_ALIGNED(size, pitch)); xe_gt_assert(gt, pitch <= U16_MAX); @@ -788,7 +808,9 @@ static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, xe_gt_assert(gt, width <= U16_MAX); - bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type; + len = blt_mem_copy_cmd_len(gt_to_xe(gt)); + + bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type | (len - 2); bb->cs[bb->len++] = width - 1; bb->cs[bb->len++] = size / pitch - 1; /* ignored by hw for page-copy/linear above */ bb->cs[bb->len++] = pitch - 1; @@ -965,7 +987,7 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m, } /* Add copy commands size here */ - batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + + batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) + ((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0)); bb = xe_bb_new(gt, batch_size, usm); @@ -1166,11 +1188,12 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, u32 batch_size, batch_size_allocated; struct xe_device *xe = gt_to_xe(gt); struct xe_res_cursor src_it, ccs_it; + struct xe_mem_pool *bb_pool; struct xe_sriov_vf_ccs_ctx *ctx; - struct xe_sa_manager *bb_pool; u64 size = xe_bo_size(src_bo); - struct xe_bb *bb = NULL; + struct xe_mem_pool_node *bb; u64 src_L0, src_L0_ofs; + struct xe_bb xe_bb_tmp; u32 src_L0_pt; int err; @@ -1208,18 +1231,18 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, size -= src_L0; } - bb = xe_bb_alloc(gt); + bb = xe_mem_pool_alloc_node(); if (IS_ERR(bb)) return PTR_ERR(bb); bb_pool = ctx->mem.ccs_bb_pool; - scoped_guard(mutex, xe_sa_bo_swap_guard(bb_pool)) { - xe_sa_bo_swap_shadow(bb_pool); + scoped_guard(mutex, xe_mem_pool_bo_swap_guard(bb_pool)) { + xe_mem_pool_swap_shadow_locked(bb_pool); - err = xe_bb_init(bb, bb_pool, batch_size); + err = xe_mem_pool_insert_node(bb_pool, bb, batch_size * sizeof(u32)); if (err) { xe_gt_err(gt, "BB allocation failed.\n"); - xe_bb_free(bb, NULL); + kfree(bb); return err; } @@ -1227,6 +1250,7 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, size = xe_bo_size(src_bo); batch_size = 0; + xe_bb_tmp = (struct xe_bb){ .cs = xe_mem_pool_node_cpu_addr(bb), .len = 0 }; /* * Emit PTE and copy commands here. * The CCS copy command can only support limited size. If the size to be @@ -1255,24 +1279,27 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); batch_size += EMIT_COPY_CCS_DW; - emit_pte(m, bb, src_L0_pt, false, true, &src_it, src_L0, src); + emit_pte(m, &xe_bb_tmp, src_L0_pt, false, true, &src_it, src_L0, src); - emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); + emit_pte(m, &xe_bb_tmp, ccs_pt, false, false, &ccs_it, ccs_size, src); - bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags); - flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_pltt, + xe_bb_tmp.len = emit_flush_invalidate(xe_bb_tmp.cs, xe_bb_tmp.len, + flush_flags); + flush_flags = xe_migrate_ccs_copy(m, &xe_bb_tmp, src_L0_ofs, src_is_pltt, src_L0_ofs, dst_is_pltt, src_L0, ccs_ofs, true); - bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags); + xe_bb_tmp.len = emit_flush_invalidate(xe_bb_tmp.cs, xe_bb_tmp.len, + flush_flags); size -= src_L0; } - xe_assert(xe, (batch_size_allocated == bb->len)); + xe_assert(xe, (batch_size_allocated == xe_bb_tmp.len)); + xe_assert(xe, bb->sa_node.size == xe_bb_tmp.len * sizeof(u32)); src_bo->bb_ccs[read_write] = bb; xe_sriov_vf_ccs_rw_update_bb_addr(ctx); - xe_sa_bo_sync_shadow(bb->bo); + xe_mem_pool_sync_shadow_locked(bb); } return 0; @@ -1297,10 +1324,10 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, enum xe_sriov_vf_ccs_rw_ctxs read_write) { - struct xe_bb *bb = src_bo->bb_ccs[read_write]; + struct xe_mem_pool_node *bb = src_bo->bb_ccs[read_write]; struct xe_device *xe = xe_bo_device(src_bo); + struct xe_mem_pool *bb_pool; struct xe_sriov_vf_ccs_ctx *ctx; - struct xe_sa_manager *bb_pool; u32 *cs; xe_assert(xe, IS_SRIOV_VF(xe)); @@ -1308,17 +1335,17 @@ void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, ctx = &xe->sriov.vf.ccs.contexts[read_write]; bb_pool = ctx->mem.ccs_bb_pool; - guard(mutex) (xe_sa_bo_swap_guard(bb_pool)); - xe_sa_bo_swap_shadow(bb_pool); + scoped_guard(mutex, xe_mem_pool_bo_swap_guard(bb_pool)) { + xe_mem_pool_swap_shadow_locked(bb_pool); - cs = xe_sa_bo_cpu_addr(bb->bo); - memset(cs, MI_NOOP, bb->len * sizeof(u32)); - xe_sriov_vf_ccs_rw_update_bb_addr(ctx); - - xe_sa_bo_sync_shadow(bb->bo); + cs = xe_mem_pool_node_cpu_addr(bb); + memset(cs, MI_NOOP, bb->sa_node.size); + xe_sriov_vf_ccs_rw_update_bb_addr(ctx); - xe_bb_free(bb, NULL); - src_bo->bb_ccs[read_write] = NULL; + xe_mem_pool_sync_shadow_locked(bb); + xe_mem_pool_free_node(bb); + src_bo->bb_ccs[read_write] = NULL; + } } /** @@ -1399,7 +1426,7 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off batch_size += pte_update_size(m, 0, sysmem, &sysmem_it, &vram_L0, &sysmem_L0_ofs, &sysmem_L0_pt, 0, avail_pts, avail_pts); - batch_size += EMIT_COPY_DW; + batch_size += emit_copy_cmd_len(xe); bb = xe_bb_new(gt, batch_size, usm); if (IS_ERR(bb)) { @@ -1454,12 +1481,17 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off return fence; } +static u32 blt_mem_set_cmd_len(struct xe_device *xe) +{ + return 7; +} + static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch) { struct xe_device *xe = gt_to_xe(gt); u32 *cs = bb->cs + bb->len; - u32 len = PVC_MEM_SET_CMD_LEN_DW; + u32 len = blt_mem_set_cmd_len(xe); *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); *cs++ = pitch - 1; @@ -1477,15 +1509,21 @@ static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs bb->len += len; } +static u32 blt_fast_color_cmd_len(struct xe_device *xe) +{ + if (GRAPHICS_VERx100(xe) >= 1250) + return 16; + else + return 11; +} + static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch, bool is_vram) { struct xe_device *xe = gt_to_xe(gt); u32 *cs = bb->cs + bb->len; - u32 len = XY_FAST_COLOR_BLT_DW; + u32 len = blt_fast_color_cmd_len(xe); - if (GRAPHICS_VERx100(xe) < 1250) - len = 11; *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | (len - 2); @@ -1518,32 +1556,20 @@ static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, bb->len += len; } -static bool has_service_copy_support(struct xe_gt *gt) -{ - /* - * What we care about is whether the architecture was designed with - * service copy functionality (specifically the new MEM_SET / MEM_COPY - * instructions) so check the architectural engine list rather than the - * actual list since these instructions are usable on BCS0 even if - * all of the actual service copy engines (BCS1-BCS8) have been fused - * off. - */ - return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8, - XE_HW_ENGINE_BCS1); -} - static u32 emit_clear_cmd_len(struct xe_gt *gt) { - if (has_service_copy_support(gt)) - return PVC_MEM_SET_CMD_LEN_DW; + struct xe_device *xe = gt_to_xe(gt); + + if (gt->info.has_xe2_blt_instructions) + return blt_mem_set_cmd_len(xe); else - return XY_FAST_COLOR_BLT_DW; + return blt_fast_color_cmd_len(xe); } static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, u32 size, u32 pitch, bool is_vram) { - if (has_service_copy_support(gt)) + if (gt->info.has_xe2_blt_instructions) emit_clear_link_copy(gt, bb, src_ofs, size, pitch); else emit_clear_main_copy(gt, bb, src_ofs, size, pitch, @@ -1870,7 +1896,7 @@ __xe_migrate_update_pgtables(struct xe_migrate *m, /* For sysmem PTE's, need to map them in our hole.. */ if (!IS_DGFX(xe)) { - u16 pat_index = xe->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); u32 ptes, ofs; ppgtt_ofs = NUM_KERNEL_PDE - 1; @@ -2092,7 +2118,7 @@ static void build_pt_update_batch_sram(struct xe_migrate *m, struct drm_pagemap_addr *sram_addr, u32 size, int level) { - u16 pat_index = tile_to_xe(m->tile)->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(tile_to_xe(m->tile), XE_CACHE_WB); u64 gpu_page_size = 0x1ull << xe_pt_shift(level); u32 ptes; int i = 0; @@ -2210,7 +2236,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, xe_assert(xe, npages * PAGE_SIZE <= MAX_PREEMPTDISABLE_TRANSFER); batch_size += pte_update_cmd_size(npages << PAGE_SHIFT); - batch_size += EMIT_COPY_DW; + batch_size += emit_copy_cmd_len(xe); bb = xe_bb_new(gt, batch_size, use_usm_batch); if (IS_ERR(bb)) { diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 169279d9d8c2..965c45889c72 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -3,8 +3,8 @@ * Copyright © 2020 Intel Corporation */ -#ifndef _XE_MIGRATE_ -#define _XE_MIGRATE_ +#ifndef _XE_MIGRATE_H_ +#define _XE_MIGRATE_H_ #include <linux/types.h> diff --git a/drivers/gpu/drm/xe/xe_nvm.h b/drivers/gpu/drm/xe/xe_nvm.h index fd3467ad35a4..b14722103f81 100644 --- a/drivers/gpu/drm/xe/xe_nvm.h +++ b/drivers/gpu/drm/xe/xe_nvm.h @@ -3,8 +3,8 @@ * Copyright(c) 2019-2025 Intel Corporation. All rights reserved. */ -#ifndef __XE_NVM_H__ -#define __XE_NVM_H__ +#ifndef _XE_NVM_H_ +#define _XE_NVM_H_ struct xe_device; diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 6337e671c97a..4bf4b1f65929 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -9,11 +9,13 @@ #include <linux/poll.h> #include <drm/drm_drv.h> +#include <drm/drm_gem.h> #include <drm/drm_managed.h> #include <drm/drm_syncobj.h> #include <uapi/drm/xe_drm.h> #include <generated/xe_wa_oob.h> +#include <generated/xe_device_wa_oob.h> #include "abi/guc_actions_slpc_abi.h" #include "instructions/xe_mi_commands.h" @@ -213,32 +215,45 @@ static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream) #define oa_report_header_64bit(__s) \ ((__s)->oa_buffer.format->header == HDR_64_BIT) -static u64 oa_report_id(struct xe_oa_stream *stream, void *report) +static u64 oa_report_id(struct xe_oa_stream *stream, u32 report_offset) { - return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report; + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + return oa_report_header_64bit(stream) ? + xe_map_rd(stream->oa->xe, map, report_offset, u64) : + xe_map_rd(stream->oa->xe, map, report_offset, u32); } -static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report) +static void oa_report_id_clear(struct xe_oa_stream *stream, u32 report_offset) { - if (oa_report_header_64bit(stream)) - *(u64 *)report = 0; - else - *report = 0; + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + oa_report_header_64bit(stream) ? + xe_map_wr(stream->oa->xe, map, report_offset, u64, 0) : + xe_map_wr(stream->oa->xe, map, report_offset, u32, 0); } -static u64 oa_timestamp(struct xe_oa_stream *stream, void *report) +static u64 oa_timestamp(struct xe_oa_stream *stream, u32 report_offset) { + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + return oa_report_header_64bit(stream) ? - *((u64 *)report + 1) : - *((u32 *)report + 1); + xe_map_rd(stream->oa->xe, map, report_offset + 8, u64) : + xe_map_rd(stream->oa->xe, map, report_offset + 4, u32); } -static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report) +static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 report_offset) { - if (oa_report_header_64bit(stream)) - *(u64 *)&report[2] = 0; - else - report[1] = 0; + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + + oa_report_header_64bit(stream) ? + xe_map_wr(stream->oa->xe, map, report_offset + 8, u64, 0) : + xe_map_wr(stream->oa->xe, map, report_offset + 4, u32, 0); +} + +static bool mert_wa_14026633728(struct xe_oa_stream *s) +{ + return s->oa_unit->type == DRM_XE_OA_UNIT_TYPE_MERT && XE_DEVICE_WA(s->oa->xe, 14026633728); } static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) @@ -275,9 +290,7 @@ static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) * they were written. If not : (╯°□°）╯︵ ┻━┻ */ while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) { - void *report = stream->oa_buffer.vaddr + tail; - - if (oa_report_id(stream, report) || oa_timestamp(stream, report)) + if (oa_report_id(stream, tail) || oa_timestamp(stream, tail)) break; tail = xe_oa_circ_diff(stream, tail, report_size); @@ -311,30 +324,37 @@ static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer) return HRTIMER_RESTART; } +static unsigned long +xe_oa_copy_to_user(struct xe_oa_stream *stream, void __user *dst, u32 report_offset, u32 len) +{ + xe_assert(stream->oa->xe, len <= stream->oa_buffer.format->size); + + xe_map_memcpy_from(stream->oa->xe, stream->oa_buffer.bounce, + &stream->oa_buffer.bo->vmap, report_offset, len); + return copy_to_user(dst, stream->oa_buffer.bounce, len); +} + static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf, - size_t count, size_t *offset, const u8 *report) + size_t count, size_t *offset, u32 report_offset) { int report_size = stream->oa_buffer.format->size; int report_size_partial; - u8 *oa_buf_end; if ((count - *offset) < report_size) return -ENOSPC; buf += *offset; - oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; - report_size_partial = oa_buf_end - report; + report_size_partial = stream->oa_buffer.circ_size - report_offset; if (report_size_partial < report_size) { - if (copy_to_user(buf, report, report_size_partial)) + if (xe_oa_copy_to_user(stream, buf, report_offset, report_size_partial)) return -EFAULT; buf += report_size_partial; - if (copy_to_user(buf, stream->oa_buffer.vaddr, - report_size - report_size_partial)) + if (xe_oa_copy_to_user(stream, buf, 0, report_size - report_size_partial)) return -EFAULT; - } else if (copy_to_user(buf, report, report_size)) { + } else if (xe_oa_copy_to_user(stream, buf, report_offset, report_size)) { return -EFAULT; } @@ -347,7 +367,6 @@ static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, size_t count, size_t *offset) { int report_size = stream->oa_buffer.format->size; - u8 *oa_buf_base = stream->oa_buffer.vaddr; u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); size_t start_offset = *offset; unsigned long flags; @@ -364,26 +383,24 @@ static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, for (; xe_oa_circ_diff(stream, tail, head); head = xe_oa_circ_incr(stream, head, report_size)) { - u8 *report = oa_buf_base + head; - - ret = xe_oa_append_report(stream, buf, count, offset, report); + ret = xe_oa_append_report(stream, buf, count, offset, head); if (ret) break; if (!(stream->oa_buffer.circ_size % report_size)) { /* Clear out report id and timestamp to detect unlanded reports */ - oa_report_id_clear(stream, (void *)report); - oa_timestamp_clear(stream, (void *)report); + oa_report_id_clear(stream, head); + oa_timestamp_clear(stream, head); } else { - u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; - u32 part = oa_buf_end - report; + struct iosys_map *map = &stream->oa_buffer.bo->vmap; + u32 part = stream->oa_buffer.circ_size - head; /* Zero out the entire report */ if (report_size <= part) { - memset(report, 0, report_size); + xe_map_memset(stream->oa->xe, map, head, 0, report_size); } else { - memset(report, 0, part); - memset(oa_buf_base, 0, report_size - part); + xe_map_memset(stream->oa->xe, map, head, 0, part); + xe_map_memset(stream->oa->xe, map, 0, 0, report_size - part); } } } @@ -436,7 +453,8 @@ static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream) spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); /* Zero out the OA buffer since we rely on zero report id and timestamp fields */ - memset(stream->oa_buffer.vaddr, 0, xe_bo_size(stream->oa_buffer.bo)); + xe_map_memset(stream->oa->xe, &stream->oa_buffer.bo->vmap, 0, 0, + xe_bo_size(stream->oa_buffer.bo)); } static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask) @@ -699,6 +717,7 @@ static int num_lri_dwords(int num_regs) static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream) { xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); + kfree(stream->oa_buffer.bounce); } static void xe_oa_free_configs(struct xe_oa_stream *stream) @@ -880,18 +899,25 @@ static void xe_oa_stream_destroy(struct xe_oa_stream *stream) static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size) { + u32 vram = mert_wa_14026633728(stream) ? + XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(stream->oa->xe)) : + XE_BO_FLAG_SYSTEM; struct xe_bo *bo; bo = xe_bo_create_pin_map_novm(stream->oa->xe, stream->gt->tile, size, ttm_bo_type_kernel, - XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT, false); + vram | XE_BO_FLAG_GGTT, false); if (IS_ERR(bo)) return PTR_ERR(bo); stream->oa_buffer.bo = bo; - /* mmap implementation requires OA buffer to be in system memory */ - xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0); - stream->oa_buffer.vaddr = bo->vmap.vaddr; + + stream->oa_buffer.bounce = kmalloc(stream->oa_buffer.format->size, GFP_KERNEL); + if (!stream->oa_buffer.bounce) { + xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); + return -ENOMEM; + } + return 0; } @@ -1673,8 +1699,6 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) { struct xe_oa_stream *stream = file->private_data; struct xe_bo *bo = stream->oa_buffer.bo; - unsigned long start = vma->vm_start; - int i, ret; if (xe_observation_paranoid && !perfmon_capable()) { drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); @@ -1682,7 +1706,7 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) } /* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */ - if (vma->vm_end - vma->vm_start != xe_bo_size(stream->oa_buffer.bo)) { + if (vma->vm_end - vma->vm_start != xe_bo_size(bo)) { drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n"); return -EINVAL; } @@ -1698,17 +1722,7 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY, VM_MAYWRITE | VM_MAYEXEC); - xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma)); - for (i = 0; i < bo->ttm.ttm->num_pages; i++) { - ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]), - PAGE_SIZE, vma->vm_page_prot); - if (ret) - break; - - start += PAGE_SIZE; - } - - return ret; + return drm_gem_mmap_obj(&bo->ttm.base, xe_bo_size(bo), vma); } static const struct file_operations xe_oa_fops = { @@ -1920,16 +1934,21 @@ static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent) return div_u64(nom + den - 1, den); } -static bool oa_unit_supports_oa_format(struct xe_oa_open_param *param, int type) +static bool oa_unit_supports_oa_format(struct xe_oa *oa, struct xe_oa_open_param *param) { + const struct xe_oa_format *f = &oa->oa_formats[param->oa_format]; + switch (param->oa_unit->type) { case DRM_XE_OA_UNIT_TYPE_OAG: - return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR || - type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC; + return f->type == DRM_XE_OA_FMT_TYPE_OAG || f->type == DRM_XE_OA_FMT_TYPE_OAR || + f->type == DRM_XE_OA_FMT_TYPE_OAC || f->type == DRM_XE_OA_FMT_TYPE_PEC; + case DRM_XE_OA_UNIT_TYPE_MERT: + if (XE_DEVICE_WA(oa->xe, 14026746987)) + return param->oa_format == XE_OAM_FORMAT_MPEC8u32_B8_C8; + fallthrough; case DRM_XE_OA_UNIT_TYPE_OAM: case DRM_XE_OA_UNIT_TYPE_OAM_SAG: - case DRM_XE_OA_UNIT_TYPE_MERT: - return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; + return f->type == DRM_XE_OA_FMT_TYPE_OAM || f->type == DRM_XE_OA_FMT_TYPE_OAM_MPEC; default: return false; } @@ -2032,8 +2051,10 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f if (XE_IOCTL_DBG(oa->xe, !param.exec_q)) return -ENOENT; - if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1)) - return -EOPNOTSUPP; + if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1)) { + ret = -EOPNOTSUPP; + goto err_exec_q; + } } /* @@ -2069,8 +2090,7 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f goto err_exec_q; f = &oa->oa_formats[param.oa_format]; - if (!param.oa_format || !f->size || - !oa_unit_supports_oa_format(¶m, f->type)) { + if (!param.oa_format || !f->size || !oa_unit_supports_oa_format(oa, ¶m)) { drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n", param.oa_format, f->type, f->size, param.hwe->class); ret = -EINVAL; @@ -2231,15 +2251,19 @@ static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr) return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs); } -static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr) +static bool xe_oa_is_valid_config_reg(struct xe_oa *oa, u32 addr, u32 val) { + if (XE_DEVICE_WA(oa->xe, 14026779378) && + addr == SYS_MEM_LAT_MEASURE.addr && val & SYS_MEM_LAT_MEASURE_EN) + return false; + return xe_oa_is_valid_flex_addr(oa, addr) || xe_oa_is_valid_b_counter_addr(oa, addr) || xe_oa_is_valid_mux_addr(oa, addr); } static struct xe_oa_reg * -xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr), +xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr, u32 val), u32 __user *regs, u32 n_regs) { struct xe_oa_reg *oa_regs; @@ -2257,16 +2281,16 @@ xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr), if (err) goto addr_err; - if (!is_valid(oa, addr)) { - drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr); - err = -EINVAL; - goto addr_err; - } - err = get_user(value, regs + 1); if (err) goto addr_err; + if (!is_valid(oa, addr, value)) { + drm_dbg(&oa->xe->drm, "Invalid oa_reg addr/value: %#x %#x\n", addr, value); + err = -EINVAL; + goto addr_err; + } + oa_regs[i].addr = XE_REG(addr); oa_regs[i].value = value; @@ -2365,7 +2389,7 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid)); oa_config->regs_len = arg->n_regs; - regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr, + regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg, u64_to_user_ptr(arg->regs_ptr), arg->n_regs); if (IS_ERR(regs)) { diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h index b03ffd513483..3d9ec8490899 100644 --- a/drivers/gpu/drm/xe/xe_oa_types.h +++ b/drivers/gpu/drm/xe/xe_oa_types.h @@ -67,7 +67,7 @@ struct xe_oa_format { u32 counter_select; /** @size: record size as written by HW (multiple of 64 byte cachelines) */ int size; - /** @type: of enum @drm_xe_oa_format_type */ + /** @type: of enum drm_xe_oa_format_type */ int type; /** @header: 32 or 64 bit report headers */ enum xe_oa_report_header header; @@ -154,16 +154,18 @@ struct xe_oa { u16 oa_unit_ids; }; -/** @xe_oa_buffer: State of the stream OA buffer */ +/** + * struct xe_oa_buffer - State of the stream OA buffer + */ struct xe_oa_buffer { /** @format: data format */ const struct xe_oa_format *format; - /** @format: xe_bo backing the OA buffer */ + /** @bo: xe_bo backing the OA buffer */ struct xe_bo *bo; - /** @vaddr: mapped vaddr of the OA buffer */ - u8 *vaddr; + /** @bounce: bounce buffer used with xe_map layer */ + void *bounce; /** @ptr_lock: Lock protecting reads/writes to head/tail pointers */ spinlock_t ptr_lock; diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h index b3289219b1be..c4ee625b93dd 100644 --- a/drivers/gpu/drm/xe/xe_pagefault_types.h +++ b/drivers/gpu/drm/xe/xe_pagefault_types.h @@ -86,7 +86,7 @@ struct xe_pagefault { u8 engine_class; /** @consumer.engine_instance: engine instance */ u8 engine_instance; - /** consumer.reserved: reserved bits for future expansion */ + /** @consumer.reserved: reserved bits for future expansion */ u64 reserved; } consumer; /** @@ -112,7 +112,7 @@ struct xe_pagefault { }; /** - * struct xe_pagefault_queue: Xe pagefault queue (consumer) + * struct xe_pagefault_queue - Xe pagefault queue (consumer) * * Used to capture all device page faults for deferred processing. Size this * queue to absorb the device’s worst-case number of outstanding faults. diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 356f53bdb83c..fad5b5a5ed4a 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -531,6 +531,14 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) drm_printf(p, "Page Table Access:\n"); xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); + if (xe_gt_is_media_type(gt)) + pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_ATS)); + else + pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_ATS)); + + drm_printf(p, "PCIe ATS/PASID:\n"); + xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); + return 0; } @@ -551,6 +559,7 @@ static const struct xe_pat_ops xe3p_xpc_pat_ops = { void xe_pat_init_early(struct xe_device *xe) { xe->pat.idx[XE_CACHE_WB_COMPRESSION] = XE_PAT_INVALID_IDX; + xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = XE_PAT_INVALID_IDX; if (GRAPHICS_VERx100(xe) == 3511) { xe->pat.ops = &xe3p_xpc_pat_ops; xe->pat.table = xe3p_xpc_pat_table; diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h index a1e287c08f57..7060f66e1d63 100644 --- a/drivers/gpu/drm/xe/xe_pat.h +++ b/drivers/gpu/drm/xe/xe_pat.h @@ -82,4 +82,12 @@ bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index); */ u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index); +#define xe_cache_pat_idx(xe, cache_mode) ({ \ + const struct xe_device *__xedev = (xe); \ + enum xe_cache_level __mode = (cache_mode); \ + xe_assert(__xedev, __mode < __XE_CACHE_LEVEL_COUNT); \ + xe_assert(__xedev, __xedev->pat.idx[__mode] != XE_PAT_INVALID_IDX); \ + __xedev->pat.idx[__mode]; \ +}) + #endif diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 01673d2b2464..1243c7d8ed10 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -118,6 +118,7 @@ static const struct xe_graphics_desc graphics_xe2 = { static const struct xe_graphics_desc graphics_xe3p_lpg = { XE2_GFX_FEATURES, + .has_indirect_ring_state = 1, .multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) | BIT(XE_ENGINE_CLASS_COMPUTE), .num_geometry_xecore_fuse_regs = 3, .num_compute_xecore_fuse_regs = 3, @@ -443,6 +444,7 @@ static const struct xe_device_desc nvls_desc = { .has_display = true, .has_flat_ccs = 1, .has_pre_prod_wa = 1, + .has_sriov = true, .max_gt_per_tile = 2, MULTI_LRC_MASK, .require_force_probe = true, @@ -465,6 +467,7 @@ static const struct xe_device_desc cri_desc = { .has_soc_remapper_sysctrl = true, .has_soc_remapper_telem = true, .has_sriov = true, + .has_sysctrl = true, .max_gt_per_tile = 2, MULTI_LRC_MASK, .require_force_probe = true, @@ -480,6 +483,7 @@ static const struct xe_device_desc nvlp_desc = { .has_flat_ccs = 1, .has_page_reclaim_hw_assist = true, .has_pre_prod_wa = true, + .has_sriov = true, .max_gt_per_tile = 2, MULTI_LRC_MASK, .require_force_probe = true, @@ -763,8 +767,8 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem; xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) && desc->has_sriov; + xe->info.has_sysctrl = desc->has_sysctrl; xe->info.skip_guc_pc = desc->skip_guc_pc; - xe->info.skip_mtcfg = desc->skip_mtcfg; xe->info.skip_pcode = desc->skip_pcode; xe->info.needs_scratch = desc->needs_scratch; xe->info.needs_shared_vf_gt_wq = desc->needs_shared_vf_gt_wq; @@ -806,9 +810,6 @@ static void xe_info_probe_tile_count(struct xe_device *xe) if (xe->info.tile_count == 1) return; - if (xe->info.skip_mtcfg) - return; - mmio = xe_root_tile_mmio(xe); /* @@ -851,6 +852,15 @@ static struct xe_gt *alloc_primary_gt(struct xe_tile *tile, gt->info.num_compute_xecore_fuse_regs = graphics_desc->num_compute_xecore_fuse_regs; /* + * Even if the service copy engines wind up being fused off, their + * presence in the IP descriptor indicates that the platform supports + * Xe2-style MEM_SET and MEM_COPY functionality. + */ + if (graphics_desc->hw_engine_mask & GENMASK(XE_HW_ENGINE_BCS8, + XE_HW_ENGINE_BCS1)) + gt->info.has_xe2_blt_instructions = true; + + /* * Before media version 13, the media IP was part of the primary GT * so we need to add the media engines to the primary GT's engine list. */ @@ -959,6 +969,12 @@ static int xe_info_init(struct xe_device *xe, xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp; xe->info.has_mem_copy_instr = GRAPHICS_VER(xe) >= 20; + if (IS_SRIOV_VF(xe)) { + xe->info.has_sysctrl = 0; + xe->info.has_soc_remapper_sysctrl = 0; + xe->info.has_soc_remapper_telem = 0; + } + xe_info_probe_tile_count(xe); for_each_remote_tile(tile, xe, id) { diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index 3fd22034f03e..35e6b53e18ce 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -191,6 +191,8 @@ static int pf_disable_vfs(struct xe_device *xe) pci_disable_sriov(pdev); + xe_sriov_pf_reprovision_default(xe); + pf_reset_vfs(xe, num_vfs); xe_sriov_pf_unprovision_vfs(xe, num_vfs); diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index 8eee4fb1c57c..5b85e2c24b7b 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -57,9 +57,9 @@ struct xe_device_desc { u8 has_soc_remapper_sysctrl:1; u8 has_soc_remapper_telem:1; u8 has_sriov:1; + u8 has_sysctrl:1; u8 needs_scratch:1; u8 skip_guc_pc:1; - u8 skip_mtcfg:1; u8 skip_pcode:1; u8 needs_shared_vf_gt_wq:1; }; diff --git a/drivers/gpu/drm/xe/xe_pcode_api.h b/drivers/gpu/drm/xe/xe_pcode_api.h index 85cc7478b787..94575c476e3d 100644 --- a/drivers/gpu/drm/xe/xe_pcode_api.h +++ b/drivers/gpu/drm/xe/xe_pcode_api.h @@ -3,6 +3,9 @@ * Copyright © 2022 Intel Corporation */ +#ifndef _XE_PCODE_API_H_ +#define _XE_PCODE_API_H_ + /* Internal to xe_pcode */ #include "regs/xe_reg_defs.h" @@ -47,8 +50,9 @@ #define WRITE_PSYSGPU_POWER_LIMIT 0x7 #define READ_PACKAGE_POWER_LIMIT 0x8 #define WRITE_PACKAGE_POWER_LIMIT 0x9 -#define READ_PL_FROM_FW 0x1 #define READ_PL_FROM_PCODE 0x0 +#define READ_PL_FROM_FW 0x1 +#define READ_PL_ACCEPTED 0x2 #define PCODE_THERMAL_INFO 0x25 #define READ_THERMAL_LIMITS 0x0 @@ -101,3 +105,5 @@ #define BMG_PCIE_CAP XE_REG(0x138340) #define LINK_DOWNGRADE REG_GENMASK(1, 0) #define DOWNGRADE_CAPABLE 2 + +#endif diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index 01185f10a883..d4672eb07476 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -26,6 +26,7 @@ #include "xe_pcode.h" #include "xe_pxp.h" #include "xe_sriov_vf_ccs.h" +#include "xe_sysctrl.h" #include "xe_trace.h" #include "xe_vm.h" #include "xe_wa.h" @@ -259,6 +260,8 @@ int xe_pm_resume(struct xe_device *xe) xe_i2c_pm_resume(xe, true); + xe_sysctrl_pm_resume(xe); + xe_irq_resume(xe); for_each_gt(gt, xe, id) { @@ -670,6 +673,9 @@ int xe_pm_runtime_resume(struct xe_device *xe) xe_i2c_pm_resume(xe, xe->d3cold.allowed); + if (xe->d3cold.allowed) + xe_sysctrl_pm_resume(xe); + xe_irq_resume(xe); for_each_gt(gt, xe, id) { diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 8e5f4f0dea3f..2669ff5ee747 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -14,6 +14,7 @@ #include "xe_gt_stats.h" #include "xe_migrate.h" #include "xe_page_reclaim.h" +#include "xe_pat.h" #include "xe_pt_types.h" #include "xe_pt_walk.h" #include "xe_res_cursor.h" @@ -62,7 +63,7 @@ static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm, unsigned int level) { struct xe_device *xe = tile_to_xe(tile); - u16 pat_index = xe->pat.idx[XE_CACHE_WB]; + u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); u8 id = tile->id; if (!xe_vm_has_scratch(vm)) diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h index 84b51d3762a4..a7d1bb708b69 100644 --- a/drivers/gpu/drm/xe/xe_pt_types.h +++ b/drivers/gpu/drm/xe/xe_pt_types.h @@ -34,7 +34,7 @@ struct xe_pt { bool rebind; bool is_compact; #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) - /** addr: Virtual address start address of the PT. */ + /** @addr: Virtual address start address of the PT. */ u64 addr; #endif }; diff --git a/drivers/gpu/drm/xe/xe_pt_walk.h b/drivers/gpu/drm/xe/xe_pt_walk.h index 5c02c244f7de..e91995fa703b 100644 --- a/drivers/gpu/drm/xe/xe_pt_walk.h +++ b/drivers/gpu/drm/xe/xe_pt_walk.h @@ -2,8 +2,8 @@ /* * Copyright © 2022 Intel Corporation */ -#ifndef __XE_PT_WALK__ -#define __XE_PT_WALK__ +#ifndef _XE_PT_WALK_H_ +#define _XE_PT_WALK_H_ #include <linux/pagewalk.h> #include <linux/types.h> @@ -34,7 +34,7 @@ struct xe_pt_walk { * changed during the walk. */ const u64 *shifts; - /** @max_level: Highest populated level in @sizes */ + /** @max_level: Highest populated level in @shifts */ unsigned int max_level; /** * @shared_pt_mode: Whether to skip all entries that are private @@ -49,7 +49,7 @@ struct xe_pt_walk { /** * typedef xe_pt_entry_fn - gpu page-table-walk callback-function - * @parent: The parent page.table. + * @parent: The parent page table. * @offset: The offset (number of entries) into the page table. * @level: The level of @parent. * @addr: The virtual address. @@ -111,14 +111,14 @@ static inline bool xe_pt_covers(u64 addr, u64 end, unsigned int level, } /** - * xe_pt_num_entries: Number of page-table entries of a given range at this + * xe_pt_num_entries - Number of page-table entries of a given range at this * level * @addr: Start address. * @end: End address. * @level: Page table level. * @walk: Walk info. * - * Return: The number of page table entries at this level between @start and + * Return: The number of page table entries at this level between @addr and * @end. */ static inline pgoff_t @@ -132,7 +132,7 @@ xe_pt_num_entries(u64 addr, u64 end, unsigned int level, } /** - * xe_pt_offset: Offset of the page-table entry for a given address. + * xe_pt_offset - Offset of the page-table entry for a given address. * @addr: The address. * @level: Page table level. * @walk: Walk info. diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index 7244090b0782..968b7e70b3f9 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -13,10 +13,12 @@ #include "xe_device_types.h" #include "xe_exec_queue.h" #include "xe_force_wake.h" +#include "xe_guc_exec_queue_types.h" #include "xe_guc_submit.h" #include "xe_gsc_proxy.h" #include "xe_gt_types.h" #include "xe_huc.h" +#include "xe_hw_engine.h" #include "xe_mmio.h" #include "xe_pm.h" #include "xe_pxp_submit.h" @@ -740,6 +742,10 @@ static void pxp_invalidate_queues(struct xe_pxp *pxp) spin_unlock_irq(&pxp->queues.lock); list_for_each_entry_safe(q, tmp, &to_clean, pxp.link) { + drm_dbg(&pxp->xe->drm, + "Killing queue due to PXP termination: eclass=%s, guc_id=%d\n", + xe_hw_engine_class_to_str(q->class), q->guc->id); + xe_exec_queue_kill(q); /* diff --git a/drivers/gpu/drm/xe/xe_pxp.h b/drivers/gpu/drm/xe/xe_pxp.h index 71a23280b900..4fb6e0afffd2 100644 --- a/drivers/gpu/drm/xe/xe_pxp.h +++ b/drivers/gpu/drm/xe/xe_pxp.h @@ -3,8 +3,8 @@ * Copyright(c) 2024, Intel Corporation. All rights reserved. */ -#ifndef __XE_PXP_H__ -#define __XE_PXP_H__ +#ifndef _XE_PXP_H_ +#define _XE_PXP_H_ #include <linux/types.h> @@ -32,4 +32,4 @@ int xe_pxp_key_assign(struct xe_pxp *pxp, struct xe_bo *bo); int xe_pxp_bo_key_check(struct xe_pxp *pxp, struct xe_bo *bo); int xe_pxp_obj_key_check(struct drm_gem_object *obj); -#endif /* __XE_PXP_H__ */ +#endif /* _XE_PXP_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_debugfs.h b/drivers/gpu/drm/xe/xe_pxp_debugfs.h index 988466aad50b..2997de0c90b2 100644 --- a/drivers/gpu/drm/xe/xe_pxp_debugfs.h +++ b/drivers/gpu/drm/xe/xe_pxp_debugfs.h @@ -3,11 +3,11 @@ * Copyright © 2024 Intel Corporation */ -#ifndef __XE_PXP_DEBUGFS_H__ -#define __XE_PXP_DEBUGFS_H__ +#ifndef _XE_PXP_DEBUGFS_H_ +#define _XE_PXP_DEBUGFS_H_ struct xe_pxp; void xe_pxp_debugfs_register(struct xe_pxp *pxp); -#endif /* __XE_PXP_DEBUGFS_H__ */ +#endif /* _XE_PXP_DEBUGFS_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_submit.h b/drivers/gpu/drm/xe/xe_pxp_submit.h index c9efda02f4b0..dbbbe6b92bb2 100644 --- a/drivers/gpu/drm/xe/xe_pxp_submit.h +++ b/drivers/gpu/drm/xe/xe_pxp_submit.h @@ -3,8 +3,8 @@ * Copyright(c) 2024, Intel Corporation. All rights reserved. */ -#ifndef __XE_PXP_SUBMIT_H__ -#define __XE_PXP_SUBMIT_H__ +#ifndef _XE_PXP_SUBMIT_H_ +#define _XE_PXP_SUBMIT_H_ #include <linux/types.h> @@ -19,4 +19,4 @@ int xe_pxp_submit_session_termination(struct xe_pxp *pxp, u32 id); int xe_pxp_submit_session_invalidation(struct xe_pxp_gsc_client_resources *gsc_res, u32 id); -#endif /* __XE_PXP_SUBMIT_H__ */ +#endif /* _XE_PXP_SUBMIT_H_ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_types.h b/drivers/gpu/drm/xe/xe_pxp_types.h index 53e9d48d10fb..ec86306e16f4 100644 --- a/drivers/gpu/drm/xe/xe_pxp_types.h +++ b/drivers/gpu/drm/xe/xe_pxp_types.h @@ -3,8 +3,8 @@ * Copyright(c) 2024, Intel Corporation. All rights reserved. */ -#ifndef __XE_PXP_TYPES_H__ -#define __XE_PXP_TYPES_H__ +#ifndef _XE_PXP_TYPES_H_ +#define _XE_PXP_TYPES_H_ #include <linux/completion.h> #include <linux/iosys-map.h> @@ -132,4 +132,4 @@ struct xe_pxp { u32 last_suspend_key_instance; }; -#endif /* __XE_PXP_TYPES_H__ */ +#endif /* _XE_PXP_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index d84d6a422c45..8c7d54498f38 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -231,6 +231,9 @@ static size_t calc_mem_regions_size(struct xe_device *xe) u32 num_managers = 1; int i; + if (xe_device_is_admin_only(xe)) + return sizeof(struct drm_xe_query_mem_regions); + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) if (ttm_manager_type(&xe->ttm, i)) num_managers++; @@ -259,6 +262,9 @@ static int query_mem_regions(struct xe_device *xe, if (XE_IOCTL_DBG(xe, !mem_regions)) return -ENOMEM; + if (xe_device_is_admin_only(xe)) + goto user_copy; + man = ttm_manager_type(&xe->ttm, XE_PL_TT); mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; /* @@ -297,6 +303,7 @@ static int query_mem_regions(struct xe_device *xe, } } +user_copy: if (!copy_to_user(query_ptr, mem_regions, size)) ret = 0; else diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c new file mode 100644 index 000000000000..4cb16b419b0c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_printk.h" +#include "xe_ras.h" +#include "xe_ras_types.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_event_types.h" + +/* Severity of detected errors */ +enum xe_ras_severity { + XE_RAS_SEV_NOT_SUPPORTED = 0, + XE_RAS_SEV_CORRECTABLE, + XE_RAS_SEV_UNCORRECTABLE, + XE_RAS_SEV_INFORMATIONAL, + XE_RAS_SEV_MAX +}; + +/* Major IP blocks/components where errors can originate */ +enum xe_ras_component { + XE_RAS_COMP_NOT_SUPPORTED = 0, + XE_RAS_COMP_DEVICE_MEMORY, + XE_RAS_COMP_CORE_COMPUTE, + XE_RAS_COMP_RESERVED, + XE_RAS_COMP_PCIE, + XE_RAS_COMP_FABRIC, + XE_RAS_COMP_SOC_INTERNAL, + XE_RAS_COMP_MAX +}; + +static const char *const xe_ras_severities[] = { + [XE_RAS_SEV_NOT_SUPPORTED] = "Not Supported", + [XE_RAS_SEV_CORRECTABLE] = "Correctable Error", + [XE_RAS_SEV_UNCORRECTABLE] = "Uncorrectable Error", + [XE_RAS_SEV_INFORMATIONAL] = "Informational Error", +}; +static_assert(ARRAY_SIZE(xe_ras_severities) == XE_RAS_SEV_MAX); + +static const char *const xe_ras_components[] = { + [XE_RAS_COMP_NOT_SUPPORTED] = "Not Supported", + [XE_RAS_COMP_DEVICE_MEMORY] = "Device Memory", + [XE_RAS_COMP_CORE_COMPUTE] = "Core Compute", + [XE_RAS_COMP_RESERVED] = "Reserved", + [XE_RAS_COMP_PCIE] = "PCIe", + [XE_RAS_COMP_FABRIC] = "Fabric", + [XE_RAS_COMP_SOC_INTERNAL] = "SoC Internal", +}; +static_assert(ARRAY_SIZE(xe_ras_components) == XE_RAS_COMP_MAX); + +static inline const char *sev_to_str(u8 severity) +{ + if (severity >= XE_RAS_SEV_MAX) + severity = XE_RAS_SEV_NOT_SUPPORTED; + + return xe_ras_severities[severity]; +} + +static inline const char *comp_to_str(u8 component) +{ + if (component >= XE_RAS_COMP_MAX) + component = XE_RAS_COMP_NOT_SUPPORTED; + + return xe_ras_components[component]; +} + +void xe_ras_counter_threshold_crossed(struct xe_device *xe, + struct xe_sysctrl_event_response *response) +{ + struct xe_ras_threshold_crossed *pending = (void *)&response->data; + struct xe_ras_error_class *errors = pending->counters; + u32 id, ncounters = pending->ncounters; + + BUILD_BUG_ON(sizeof(response->data) < sizeof(*pending)); + xe_device_assert_mem_access(xe); + + if (!ncounters || ncounters > XE_RAS_NUM_COUNTERS) + xe_err(xe, "sysctrl: unexpected counter threshold crossed %u\n", ncounters); + else + xe_warn(xe, "[RAS]: counter threshold crossed, %u new errors\n", ncounters); + + for (id = 0; id < ncounters && id < XE_RAS_NUM_COUNTERS; id++) { + u8 severity, component; + + severity = errors[id].common.severity; + component = errors[id].common.component; + + xe_warn(xe, "[RAS]: %s %s detected\n", + comp_to_str(component), sev_to_str(severity)); + } +} diff --git a/drivers/gpu/drm/xe/xe_ras.h b/drivers/gpu/drm/xe/xe_ras.h new file mode 100644 index 000000000000..ea90593b62dc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_RAS_H_ +#define _XE_RAS_H_ + +struct xe_device; +struct xe_sysctrl_event_response; + +void xe_ras_counter_threshold_crossed(struct xe_device *xe, + struct xe_sysctrl_event_response *response); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ras_types.h b/drivers/gpu/drm/xe/xe_ras_types.h new file mode 100644 index 000000000000..4e63c67f806a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ras_types.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_RAS_TYPES_H_ +#define _XE_RAS_TYPES_H_ + +#include <linux/types.h> + +#define XE_RAS_NUM_COUNTERS 16 + +/** + * struct xe_ras_error_common - Error fields that are common across all products + */ +struct xe_ras_error_common { + /** @severity: Error severity */ + u8 severity; + /** @component: IP block where error originated */ + u8 component; +} __packed; + +/** + * struct xe_ras_error_unit - Error unit information + */ +struct xe_ras_error_unit { + /** @tile: Tile identifier */ + u8 tile; + /** @instance: Instance identifier specific to IP */ + u32 instance; +} __packed; + +/** + * struct xe_ras_error_cause - Error cause information + */ +struct xe_ras_error_cause { + /** @cause: Cause/checker */ + u32 cause; + /** @reserved: For future use */ + u8 reserved; +} __packed; + +/** + * struct xe_ras_error_product - Error fields that are specific to the product + */ +struct xe_ras_error_product { + /** @unit: Unit within IP block */ + struct xe_ras_error_unit unit; + /** @cause: Cause/checker */ + struct xe_ras_error_cause cause; +} __packed; + +/** + * struct xe_ras_error_class - Combines common and product-specific parts + */ +struct xe_ras_error_class { + /** @common: Common error type and component */ + struct xe_ras_error_common common; + /** @product: Product-specific unit and cause */ + struct xe_ras_error_product product; +} __packed; + +/** + * struct xe_ras_threshold_crossed - Data for threshold crossed event + */ +struct xe_ras_threshold_crossed { + /** @ncounters: Number of error counters that crossed thresholds */ + u32 ncounters; + /** @counters: Array of error counters that crossed threshold */ + struct xe_ras_error_class counters[XE_RAS_NUM_COUNTERS]; +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c index 2df0277efb2f..e328f5072557 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr.c +++ b/drivers/gpu/drm/xe/xe_reg_sr.c @@ -70,14 +70,49 @@ static void reg_sr_inc_error(struct xe_reg_sr *sr) #endif } +static struct xe_reg sanitize_mcr(struct xe_reg_sr *sr, + const struct xe_reg_sr_entry *e, + struct xe_gt *gt) +{ + struct xe_reg reg = e->reg; + bool is_mcr; + + /* + * We need the gt structure to check MCR ranges. + */ + if (!gt) + return reg; + + is_mcr = xe_gt_mcr_check_reg(gt, reg); + + if (is_mcr && !reg.mcr) { + reg.mcr = 1; + xe_gt_notice(gt, "xe_reg_sr_entry using non-MCR register for address 0x%x, forcing MCR\n", + reg.addr); + reg_sr_inc_error(sr); + } + + if (!is_mcr && reg.mcr) { + reg.mcr = 0; + xe_gt_notice(gt, "xe_reg_sr_entry using MCR register for address 0x%x, forcing non-MCR\n", + reg.addr); + reg_sr_inc_error(sr); + } + + return reg; +} + int xe_reg_sr_add(struct xe_reg_sr *sr, const struct xe_reg_sr_entry *e, struct xe_gt *gt) { unsigned long idx = e->reg.addr; struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); + struct xe_reg reg; int ret; + reg = sanitize_mcr(sr, e, gt); + if (pentry) { if (!compatible_entries(pentry, e)) { ret = -EINVAL; @@ -98,6 +133,7 @@ int xe_reg_sr_add(struct xe_reg_sr *sr, } *pentry = *e; + pentry->reg = reg; ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); if (ret) goto fail_free; diff --git a/drivers/gpu/drm/xe/xe_reg_sr.h b/drivers/gpu/drm/xe/xe_reg_sr.h index 1ec6e8ecf278..d26cf4713383 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr.h +++ b/drivers/gpu/drm/xe/xe_reg_sr.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_REG_SR_ -#define _XE_REG_SR_ +#ifndef _XE_REG_SR_H_ +#define _XE_REG_SR_H_ /* * Reg save/restore bookkeeping diff --git a/drivers/gpu/drm/xe/xe_reg_sr_types.h b/drivers/gpu/drm/xe/xe_reg_sr_types.h index ebe11f237fa2..0a6695db2967 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr_types.h +++ b/drivers/gpu/drm/xe/xe_reg_sr_types.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_REG_SR_TYPES_ -#define _XE_REG_SR_TYPES_ +#ifndef _XE_REG_SR_TYPES_H_ +#define _XE_REG_SR_TYPES_H_ #include <linux/types.h> #include <linux/xarray.h> diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c index 80577e4b7437..fb65940848d7 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.c +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -9,6 +9,7 @@ #include "regs/xe_gt_regs.h" #include "regs/xe_oa_regs.h" #include "xe_device.h" +#include "xe_gt.h" #include "xe_gt_types.h" #include "xe_gt_printk.h" #include "xe_platform_types.h" @@ -33,6 +34,13 @@ static bool match_has_mert(const struct xe_device *xe, return xe_device_has_mert((struct xe_device *)xe); } +static bool match_multi_queue_class(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_gt_supports_multi_queue(gt, hwe->class); +} + static const struct xe_rtp_entry_sr register_whitelist[] = { { XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), @@ -54,6 +62,12 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { RING_FORCE_TO_NONPRIV_ACCESS_RD, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, + { XE_RTP_NAME("allow_read_queue_timestamp"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, 3511), FUNC(match_multi_queue_class)), + XE_RTP_ACTIONS(WHITELIST(RING_QUEUE_TIMESTAMP(0), + RING_FORCE_TO_NONPRIV_ACCESS_RD, + XE_RTP_ACTION_FLAG(ENGINE_BASE))) + }, { XE_RTP_NAME("16014440446"), XE_RTP_RULES(PLATFORM(PVC)), XE_RTP_ACTIONS(WHITELIST(XE_REG(0x4400), @@ -226,7 +240,7 @@ void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent, } range_start = reg & REG_GENMASK(25, range_bit); - range_end = range_start | REG_GENMASK(range_bit, 0); + range_end = range_start | REG_GENMASK(range_bit - 1, 0); switch (val & RING_FORCE_TO_NONPRIV_ACCESS_MASK) { case RING_FORCE_TO_NONPRIV_ACCESS_RW: diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.h b/drivers/gpu/drm/xe/xe_reg_whitelist.h index 69b121d377da..3b64b42fe96e 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.h +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.h @@ -3,8 +3,8 @@ * Copyright © 2023 Intel Corporation */ -#ifndef _XE_REG_WHITELIST_ -#define _XE_REG_WHITELIST_ +#ifndef _XE_REG_WHITELIST_H_ +#define _XE_REG_WHITELIST_H_ #include <linux/types.h> diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h index 5f4ab08c0686..0522caafd89d 100644 --- a/drivers/gpu/drm/xe/xe_res_cursor.h +++ b/drivers/gpu/drm/xe/xe_res_cursor.h @@ -101,7 +101,15 @@ static inline void xe_res_first(struct ttm_resource *res, cur->mem_type = res->mem_type; switch (cur->mem_type) { - case XE_PL_STOLEN: + case XE_PL_STOLEN: { + /* res->start is in pages (ttm_range_manager). */ + cur->start = (res->start << PAGE_SHIFT) + start; + cur->size = size; + cur->remaining = size; + cur->node = NULL; + cur->mm = NULL; + break; + } case XE_PL_VRAM0: case XE_PL_VRAM1: { struct gpu_buddy_block *block; @@ -289,6 +297,10 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size) switch (cur->mem_type) { case XE_PL_STOLEN: + /* Just advance within the contiguous region. */ + cur->start += size; + cur->size = cur->remaining; + break; case XE_PL_VRAM0: case XE_PL_VRAM1: start = size - cur->size; diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index cfeb4fc7d217..39a670e91ba7 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -269,8 +269,12 @@ static u32 get_ppgtt_flag(struct xe_sched_job *job) static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, u32 *dw, int i) { + const struct xe_reg reg = xe_lrc_is_multi_queue(lrc) ? + RING_QUEUE_TIMESTAMP(0) : + RING_CTX_TIMESTAMP(0); + dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; - dw[i++] = RING_CTX_TIMESTAMP(0).addr; + dw[i++] = reg.addr; dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); dw[i++] = 0; @@ -281,7 +285,7 @@ static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, if (IS_SRIOV_VF(xe)) { dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; - dw[i++] = RING_CTX_TIMESTAMP(0).addr; + dw[i++] = reg.addr; dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc); dw[i++] = 0; } diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c index 991f218f1cc3..1a4dcbbbc176 100644 --- a/drivers/gpu/drm/xe/xe_rtp.c +++ b/drivers/gpu/drm/xe/xe_rtp.c @@ -10,6 +10,7 @@ #include <uapi/drm/xe_drm.h> #include "xe_configfs.h" +#include "xe_device.h" #include "xe_gt.h" #include "xe_gt_topology.h" #include "xe_reg_sr.h" @@ -352,6 +353,13 @@ void xe_rtp_process(struct xe_rtp_process_ctx *ctx, } EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process); +bool xe_rtp_match_always(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return true; +} + bool xe_rtp_match_even_instance(const struct xe_device *xe, const struct xe_gt *gt, const struct xe_hw_engine *hwe) @@ -397,3 +405,10 @@ bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe, { return xe->info.has_flat_ccs; } + +bool xe_rtp_match_has_msix(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return xe_device_has_msix(xe); +} diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h index 7d6daa7eb1e4..562082b18d7b 100644 --- a/drivers/gpu/drm/xe/xe_rtp.h +++ b/drivers/gpu/drm/xe/xe_rtp.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_RTP_ -#define _XE_RTP_ +#ifndef _XE_RTP_H_ +#define _XE_RTP_H_ #include <linux/types.h> #include <linux/xarray.h> @@ -460,6 +460,18 @@ void xe_rtp_process(struct xe_rtp_process_ctx *ctx, /* Match functions to be used with XE_RTP_MATCH_FUNC */ /** + * xe_rtp_match_always - Match RTP entry unconditionally + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true, regardless of inputs + */ +bool xe_rtp_match_always(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + +/** * xe_rtp_match_even_instance - Match if engine instance is even * @xe: Device structure * @gt: GT structure @@ -524,4 +536,16 @@ bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe, const struct xe_gt *gt, const struct xe_hw_engine *hwe); +/** + * xe_rtp_match_has_msix - Match when platform has MSI-X + * @xe: Device structure + * @gt: GT structure + * @hwe: Engine instance + * + * Returns: true if platform has MSI-X interrupt support + */ +bool xe_rtp_match_has_msix(const struct xe_device *xe, + const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + #endif diff --git a/drivers/gpu/drm/xe/xe_rtp_helpers.h b/drivers/gpu/drm/xe/xe_rtp_helpers.h index a33b0ae98bbc..ffa8b2f8828d 100644 --- a/drivers/gpu/drm/xe/xe_rtp_helpers.h +++ b/drivers/gpu/drm/xe/xe_rtp_helpers.h @@ -3,8 +3,8 @@ * Copyright © 2023 Intel Corporation */ -#ifndef _XE_RTP_HELPERS_ -#define _XE_RTP_HELPERS_ +#ifndef _XE_RTP_HELPERS_H_ +#define _XE_RTP_HELPERS_H_ #ifndef _XE_RTP_INCLUDE_PRIVATE_HELPERS #error "This header is supposed to be included by xe_rtp.h only" @@ -66,6 +66,8 @@ #define XE_RTP_PASTE_10(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_9(prefix_, sep_, _XE_TUPLE_TAIL args_) #define XE_RTP_PASTE_11(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_10(prefix_, sep_, _XE_TUPLE_TAIL args_) #define XE_RTP_PASTE_12(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_11(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_13(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_12(prefix_, sep_, _XE_TUPLE_TAIL args_) +#define XE_RTP_PASTE_14(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_13(prefix_, sep_, _XE_TUPLE_TAIL args_) /* * XE_RTP_DROP_CAST - Drop cast to convert a compound statement to a initializer diff --git a/drivers/gpu/drm/xe/xe_rtp_types.h b/drivers/gpu/drm/xe/xe_rtp_types.h index 166251615be1..0265c16d2762 100644 --- a/drivers/gpu/drm/xe/xe_rtp_types.h +++ b/drivers/gpu/drm/xe/xe_rtp_types.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_RTP_TYPES_ -#define _XE_RTP_TYPES_ +#ifndef _XE_RTP_TYPES_H_ +#define _XE_RTP_TYPES_H_ #include <linux/types.h> diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c index 47a6e0fd66e0..33bd754d138f 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf.c @@ -20,11 +20,6 @@ #include "xe_sriov_pf_sysfs.h" #include "xe_sriov_printk.h" -static bool wanted_admin_only(struct xe_device *xe) -{ - return xe_configfs_admin_only_pf(to_pci_dev(xe->drm.dev)); -} - static unsigned int wanted_max_vfs(struct xe_device *xe) { return xe_configfs_get_max_vfs(to_pci_dev(xe->drm.dev)); @@ -79,7 +74,6 @@ bool xe_sriov_pf_readiness(struct xe_device *xe) pf_reduce_totalvfs(xe, newlimit); - xe->sriov.pf.admin_only = wanted_admin_only(xe); xe->sriov.pf.device_total_vfs = totalvfs; xe->sriov.pf.driver_max_vfs = newlimit; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h index 0fcc6cec4afc..19f6f8331c8d 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h @@ -7,6 +7,7 @@ #define _XE_SRIOV_PF_HELPERS_H_ #include "xe_assert.h" +#include "xe_device.h" #include "xe_device_types.h" #include "xe_sriov.h" #include "xe_sriov_types.h" @@ -57,7 +58,7 @@ static inline unsigned int xe_sriov_pf_num_vfs(const struct xe_device *xe) static inline bool xe_sriov_pf_admin_only(const struct xe_device *xe) { xe_assert(xe, IS_SRIOV_PF(xe)); - return xe->sriov.pf.admin_only; + return xe_device_is_admin_only(xe); } static inline struct mutex *xe_sriov_pf_master_mutex(struct xe_device *xe) diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c index 6c4b16409cc9..150a241110fb 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c @@ -149,10 +149,11 @@ pf_migration_consume(struct xe_device *xe, unsigned int vfid) for_each_gt(gt, xe, gt_id) { data = xe_gt_sriov_pf_migration_save_consume(gt, vfid); - if (data && PTR_ERR(data) != EAGAIN) + if (!data) + continue; + if (!IS_ERR(data) || PTR_ERR(data) != -EAGAIN) return data; - if (PTR_ERR(data) == -EAGAIN) - more_data = true; + more_data = true; } if (!more_data) diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c index abe3677d33ed..0ec7ea83f12a 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c @@ -41,6 +41,8 @@ static int pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs) int err; for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_config_set_fair_sched(gt, num_vfs); + result = result ?: err; err = xe_gt_sriov_pf_config_set_fair(gt, VFID(1), num_vfs); result = result ?: err; } @@ -103,6 +105,45 @@ int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs) return 0; } +static int pf_reprovision_default(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + int result = 0; + int err; + + guard(mutex)(xe_sriov_pf_master_mutex(xe)); + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, false); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_exec_quantum_locked(gt, PFID, 0); + result = result ?: err; + err = xe_gt_sriov_pf_config_set_preempt_timeout_locked(gt, PFID, 0); + result = result ?: err; + } + + return result; +} + +/** + * xe_sriov_pf_reprovision_default() - Reprovision default PF in auto-mode. + * @xe: the PF &xe_device + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_reprovision_default(struct xe_device *xe) +{ + xe_assert(xe, IS_SRIOV_PF(xe)); + + if (!pf_auto_provisioning_mode(xe)) + return 0; + + return pf_reprovision_default(xe); +} + /** * xe_sriov_pf_provision_set_mode() - Change VFs provision mode. * @xe: the PF &xe_device diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h index f26f49539697..b15e4d7ad940 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h @@ -30,6 +30,7 @@ int xe_sriov_pf_provision_query_vf_vram(struct xe_device *xe, unsigned int vfid, int xe_sriov_pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs); int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs); +int xe_sriov_pf_reprovision_default(struct xe_device *xe); int xe_sriov_pf_provision_set_mode(struct xe_device *xe, enum xe_sriov_provisioning_mode mode); diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_types.h index 080cf10512f4..b0253e1ae5da 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_types.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_types.h @@ -36,9 +36,6 @@ struct xe_sriov_metadata { * @XE_SRIOV_MODE_PF mode. */ struct xe_device_pf { - /** @admin_only: PF functionality focused on VFs management only. */ - bool admin_only; - /** @device_total_vfs: Maximum number of VFs supported by the device. */ u16 device_total_vfs; diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index db023fb66a27..09b99fb2608b 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -14,9 +14,9 @@ #include "xe_guc.h" #include "xe_guc_submit.h" #include "xe_lrc.h" +#include "xe_mem_pool.h" #include "xe_migrate.h" #include "xe_pm.h" -#include "xe_sa.h" #include "xe_sriov_printk.h" #include "xe_sriov_vf.h" #include "xe_sriov_vf_ccs.h" @@ -141,43 +141,47 @@ static u64 get_ccs_bb_pool_size(struct xe_device *xe) static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx) { + struct xe_mem_pool *pool; struct xe_device *xe = tile_to_xe(tile); - struct xe_sa_manager *sa_manager; + u32 *pool_cpu_addr, *last_dw_addr; u64 bb_pool_size; - int offset, err; + int err; bb_pool_size = get_ccs_bb_pool_size(xe); xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n", ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M); - sa_manager = __xe_sa_bo_manager_init(tile, bb_pool_size, SZ_4K, SZ_16, - XE_SA_BO_MANAGER_FLAG_SHADOW); - - if (IS_ERR(sa_manager)) { - xe_sriov_err(xe, "Suballocator init failed with error: %pe\n", - sa_manager); - err = PTR_ERR(sa_manager); + pool = xe_mem_pool_init(tile, bb_pool_size, sizeof(u32), + XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY); + if (IS_ERR(pool)) { + xe_sriov_err(xe, "xe_mem_pool_init failed with error: %pe\n", + pool); + err = PTR_ERR(pool); return err; } - offset = 0; - xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP, - bb_pool_size); - xe_map_memset(xe, &sa_manager->shadow->vmap, offset, MI_NOOP, - bb_pool_size); + pool_cpu_addr = xe_mem_pool_cpu_addr(pool); + memset(pool_cpu_addr, 0, bb_pool_size); - offset = bb_pool_size - sizeof(u32); - xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END); - xe_map_wr(xe, &sa_manager->shadow->vmap, offset, u32, MI_BATCH_BUFFER_END); + last_dw_addr = pool_cpu_addr + (bb_pool_size / sizeof(u32)) - 1; + *last_dw_addr = MI_BATCH_BUFFER_END; - ctx->mem.ccs_bb_pool = sa_manager; + /** + * Sync the main copy and shadow copy so that the shadow copy is + * replica of main copy. We sync only BBs after init part. So, we + * need to make sure the main pool and shadow copy are in sync after + * this point. This is needed as GuC may read the BB commands from + * shadow copy. + */ + xe_mem_pool_sync(pool); + ctx->mem.ccs_bb_pool = pool; return 0; } static void ccs_rw_update_ring(struct xe_sriov_vf_ccs_ctx *ctx) { - u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool); + u64 addr = xe_mem_pool_gpu_addr(ctx->mem.ccs_bb_pool); struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); u32 dw[10], i = 0; @@ -388,7 +392,7 @@ err_ret: #define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET (2 * sizeof(u32)) void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx) { - u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool); + u64 addr = xe_mem_pool_gpu_addr(ctx->mem.ccs_bb_pool); struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); struct xe_device *xe = gt_to_xe(ctx->mig_q->gt); @@ -412,8 +416,8 @@ int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) struct xe_device *xe = xe_bo_device(bo); enum xe_sriov_vf_ccs_rw_ctxs ctx_id; struct xe_sriov_vf_ccs_ctx *ctx; + struct xe_mem_pool_node *bb; struct xe_tile *tile; - struct xe_bb *bb; int err = 0; xe_assert(xe, IS_VF_CCS_READY(xe)); @@ -445,7 +449,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo) { struct xe_device *xe = xe_bo_device(bo); enum xe_sriov_vf_ccs_rw_ctxs ctx_id; - struct xe_bb *bb; + struct xe_mem_pool_node *bb; xe_assert(xe, IS_VF_CCS_READY(xe)); @@ -471,8 +475,8 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo) */ void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p) { - struct xe_sa_manager *bb_pool; enum xe_sriov_vf_ccs_rw_ctxs ctx_id; + struct xe_mem_pool *bb_pool; if (!IS_VF_CCS_READY(xe)) return; @@ -485,7 +489,7 @@ void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p) drm_printf(p, "ccs %s bb suballoc info\n", ctx_id ? "write" : "read"); drm_printf(p, "-------------------------\n"); - drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool)); + xe_mem_pool_dump(bb_pool, p); drm_puts(p, "\n"); } } diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h index 22c499943d2a..6fc8f97ef3f4 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs_types.h @@ -17,9 +17,6 @@ enum xe_sriov_vf_ccs_rw_ctxs { XE_SRIOV_VF_CCS_CTX_COUNT }; -struct xe_migrate; -struct xe_sa_manager; - /** * struct xe_sriov_vf_ccs_ctx - VF CCS migration context data. */ @@ -33,7 +30,7 @@ struct xe_sriov_vf_ccs_ctx { /** @mem: memory data */ struct { /** @mem.ccs_bb_pool: Pool from which batch buffers are allocated. */ - struct xe_sa_manager *ccs_bb_pool; + struct xe_mem_pool *ccs_bb_pool; } mem; }; diff --git a/drivers/gpu/drm/xe/xe_step.c b/drivers/gpu/drm/xe/xe_step.c index d0f888c31831..fb9c31613ca7 100644 --- a/drivers/gpu/drm/xe/xe_step.c +++ b/drivers/gpu/drm/xe/xe_step.c @@ -278,7 +278,7 @@ void xe_step_gmdid_get(struct xe_device *xe, case STEP_##name: \ return #name; -const char *xe_step_name(enum xe_step step) +const char *xe_step_name(enum intel_step step) { switch (step) { STEP_NAME_LIST(STEP_NAME_CASE); diff --git a/drivers/gpu/drm/xe/xe_step.h b/drivers/gpu/drm/xe/xe_step.h index 41f1c95c46e5..ea36b22cc297 100644 --- a/drivers/gpu/drm/xe/xe_step.h +++ b/drivers/gpu/drm/xe/xe_step.h @@ -18,8 +18,8 @@ void xe_step_pre_gmdid_get(struct xe_device *xe); void xe_step_gmdid_get(struct xe_device *xe, u32 graphics_gmdid_revid, u32 media_gmdid_revid); -static inline u32 xe_step_to_gmdid(enum xe_step step) { return step - STEP_A0; } +static inline u32 xe_step_to_gmdid(enum intel_step step) { return step - STEP_A0; } -const char *xe_step_name(enum xe_step step); +const char *xe_step_name(enum intel_step step); #endif diff --git a/drivers/gpu/drm/xe/xe_step_types.h b/drivers/gpu/drm/xe/xe_step_types.h index 43ca73850739..f60572b93523 100644 --- a/drivers/gpu/drm/xe/xe_step_types.h +++ b/drivers/gpu/drm/xe/xe_step_types.h @@ -8,6 +8,8 @@ #include <linux/types.h> +#include <drm/intel/step.h> + struct xe_step_info { u8 platform; u8 graphics; @@ -15,63 +17,4 @@ struct xe_step_info { u8 basedie; }; -#define STEP_ENUM_VAL(name) STEP_##name, - -/* - * Always define four minor steppings 0-3 for each stepping to match GMD ID - * spacing of values. See xe_step_gmdid_get(). - */ -#define STEP_NAME_LIST(func) \ - func(A0) \ - func(A1) \ - func(A2) \ - func(A3) \ - func(B0) \ - func(B1) \ - func(B2) \ - func(B3) \ - func(C0) \ - func(C1) \ - func(C2) \ - func(C3) \ - func(D0) \ - func(D1) \ - func(D2) \ - func(D3) \ - func(E0) \ - func(E1) \ - func(E2) \ - func(E3) \ - func(F0) \ - func(F1) \ - func(F2) \ - func(F3) \ - func(G0) \ - func(G1) \ - func(G2) \ - func(G3) \ - func(H0) \ - func(H1) \ - func(H2) \ - func(H3) \ - func(I0) \ - func(I1) \ - func(I2) \ - func(I3) \ - func(J0) \ - func(J1) \ - func(J2) \ - func(J3) - -/* - * Symbolic steppings that do not match the hardware. These are valid both as gt - * and display steppings as symbolic names. - */ -enum xe_step { - STEP_NONE = 0, - STEP_NAME_LIST(STEP_ENUM_VAL) - STEP_FUTURE, - STEP_FOREVER, -}; - #endif diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index db64cac39c94..427afd144f3a 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -396,25 +396,21 @@ bool xe_survivability_mode_is_requested(struct xe_device *xe) * Runtime survivability mode is enabled when certain errors cause the device to be * in non-recoverable state. The device is declared wedged with the appropriate * recovery method and survivability mode sysfs exposed to userspace - * - * Return: 0 if runtime survivability mode is enabled, negative error code otherwise. */ -int xe_survivability_mode_runtime_enable(struct xe_device *xe) +void xe_survivability_mode_runtime_enable(struct xe_device *xe) { struct xe_survivability *survivability = &xe->survivability; struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - int ret; if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || xe->info.platform < XE_BATTLEMAGE) { dev_err(&pdev->dev, "Runtime Survivability Mode not supported\n"); - return -EINVAL; + return; } populate_survivability_info(xe); - ret = create_survivability_sysfs(pdev); - if (ret) - dev_err(&pdev->dev, "Failed to create survivability mode sysfs\n"); + if (create_survivability_sysfs(pdev)) + dev_err(&pdev->dev, "Failed to create survivability sysfs\n"); survivability->type = XE_SURVIVABILITY_TYPE_RUNTIME; dev_err(&pdev->dev, "Runtime Survivability mode enabled\n"); @@ -422,8 +418,6 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe) xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR); xe_device_declare_wedged(xe); dev_err(&pdev->dev, "Firmware flash required, Please refer to the userspace documentation for more details!\n"); - - return 0; } /** diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.h b/drivers/gpu/drm/xe/xe_survivability_mode.h index 1cc94226aa82..cd040e4d18bb 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.h +++ b/drivers/gpu/drm/xe/xe_survivability_mode.h @@ -11,7 +11,7 @@ struct xe_device; int xe_survivability_mode_boot_enable(struct xe_device *xe); -int xe_survivability_mode_runtime_enable(struct xe_device *xe); +void xe_survivability_mode_runtime_enable(struct xe_device *xe); bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe); bool xe_survivability_mode_is_requested(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 5933b2b6392b..e1651e70c8f0 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -786,12 +786,12 @@ static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocati struct xe_bo *bo = to_xe_bo(devmem_allocation); struct ttm_resource *res = bo->ttm.resource; struct list_head *blocks = &to_xe_ttm_vram_mgr_resource(res)->blocks; + struct xe_vram_region *vr = xe_map_resource_to_region(res); + struct gpu_buddy *buddy = vram_to_buddy(vr); struct gpu_buddy_block *block; int j = 0; list_for_each_entry(block, blocks, link) { - struct xe_vram_region *vr = block->private; - struct gpu_buddy *buddy = vram_to_buddy(vr); u64 block_pfn = block_offset_to_pfn(devmem_allocation->dpagemap, gpu_buddy_block_offset(block)); int i; @@ -1055,15 +1055,12 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap); struct drm_pagemap_migrate_details mdetails = { .timeslice_ms = timeslice_ms, - .source_peer_migrates = 1, }; struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap); struct dma_fence *pre_migrate_fence = NULL; struct xe_device *xe = vr->xe; struct device *dev = xe->drm.dev; - struct gpu_buddy_block *block; struct xe_validation_ctx vctx; - struct list_head *blocks; struct drm_exec exec; struct xe_bo *bo; int err = 0, idx; @@ -1100,10 +1097,6 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, &dpagemap_devmem_ops, dpagemap, end - start, pre_migrate_fence); - blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks; - list_for_each_entry(block, blocks, link) - block->private = vr; - xe_bo_get(bo); /* Ensure the device has a pm ref while there are device pages active. */ diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index 24d6d9af20d6..37866768d64c 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -376,7 +376,7 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, xe_assert(vm->xe, current_fence == num_fence); cf = dma_fence_array_create(num_fence, fences, dma_fence_context_alloc(1), - 1, false); + 1); if (!cf) goto err_out; diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c new file mode 100644 index 000000000000..1db20be8158b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include <linux/device.h> +#include <linux/mutex.h> + +#include <drm/drm_managed.h> + +#include "regs/xe_irq_regs.h" +#include "regs/xe_sysctrl_regs.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_soc_remapper.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_types.h" + +/** + * DOC: System Controller (sysctrl) + * + * System Controller (sysctrl) is a firmware-managed entity on Intel dGPUs + * responsible for selected low-level platform management functions. + * Communication between driver and System Controller is performed + * via a mailbox interface, enabling command and response exchange. + * + * This module provides initialization and support code for interacting + * with System Controller through the mailbox interface. + */ +static void sysctrl_fini(void *arg) +{ + struct xe_device *xe = arg; + struct xe_sysctrl *sc = &xe->sc; + + disable_work_sync(&sc->work); + xe->soc_remapper.set_sysctrl_region(xe, 0); +} + +static void xe_sysctrl_work(struct work_struct *work) +{ + struct xe_sysctrl *sc = container_of(work, struct xe_sysctrl, work); + struct xe_device *xe = sc_to_xe(sc); + + guard(xe_pm_runtime)(xe); + xe_sysctrl_event(sc); +} + +/** + * xe_sysctrl_init() - Initialize System Controller subsystem + * @xe: xe device instance + * + * Entry point for System Controller initialization, called from xe_device_probe. + * This function checks platform support and initializes the system controller. + * + * Return: 0 on success, error code on failure + */ +int xe_sysctrl_init(struct xe_device *xe) +{ + struct xe_tile *tile = xe_device_get_root_tile(xe); + struct xe_sysctrl *sc = &xe->sc; + int ret; + + if (!xe->info.has_soc_remapper_sysctrl) + return 0; + + if (!xe->info.has_sysctrl) + return 0; + + sc->mmio = devm_kzalloc(xe->drm.dev, sizeof(*sc->mmio), GFP_KERNEL); + if (!sc->mmio) + return -ENOMEM; + + xe_mmio_init(sc->mmio, tile, tile->mmio.regs, tile->mmio.regs_size); + sc->mmio->adj_offset = SYSCTRL_BASE; + sc->mmio->adj_limit = U32_MAX; + + ret = devm_mutex_init(xe->drm.dev, &sc->cmd_lock); + if (ret) + return ret; + + ret = devm_mutex_init(xe->drm.dev, &sc->event_lock); + if (ret) + return ret; + + xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); + xe_sysctrl_mailbox_init(sc); + INIT_WORK(&sc->work, xe_sysctrl_work); + + return devm_add_action_or_reset(xe->drm.dev, sysctrl_fini, xe); +} + +/** + * xe_sysctrl_irq_handler() - Handler for System Controller interrupts + * @xe: xe device instance + * @master_ctl: interrupt register + * + * Handle interrupts generated by System Controller. + */ +void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl) +{ + struct xe_sysctrl *sc = &xe->sc; + + if (!xe->info.has_sysctrl || !sc->work.func) + return; + + if (master_ctl & SYSCTRL_IRQ) + schedule_work(&sc->work); +} + +/** + * xe_sysctrl_pm_resume() - System Controller resume handler + * @xe: xe device instance + * + * Invoked during system resume (S3/S4 to S0) and runtime resume from D3cold. + * Restores SoC remapper configuration and reinitializes mailbox interface. + */ +void xe_sysctrl_pm_resume(struct xe_device *xe) +{ + struct xe_sysctrl *sc = &xe->sc; + + if (!xe->info.has_soc_remapper_sysctrl) + return; + + if (!xe->info.has_sysctrl) + return; + + xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX); + + xe_sysctrl_mailbox_init(sc); +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl.h b/drivers/gpu/drm/xe/xe_sysctrl.h new file mode 100644 index 000000000000..090dffb6d55f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_H_ +#define _XE_SYSCTRL_H_ + +#include <linux/container_of.h> + +#include "xe_device_types.h" +#include "xe_sysctrl_types.h" + +static inline struct xe_device *sc_to_xe(struct xe_sysctrl *sc) +{ + return container_of(sc, struct xe_device, sc); +} + +void xe_sysctrl_event(struct xe_sysctrl *sc); +int xe_sysctrl_init(struct xe_device *xe); +void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl); +void xe_sysctrl_pm_resume(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event.c b/drivers/gpu/drm/xe/xe_sysctrl_event.c new file mode 100644 index 000000000000..b4d17329af6c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_event.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_irq.h" +#include "xe_printk.h" +#include "xe_ras.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_event_types.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" + +static void get_pending_event(struct xe_sysctrl *sc, struct xe_sysctrl_mailbox_command *command) +{ + struct xe_sysctrl_event_response *response = command->data_out; + struct xe_device *xe = sc_to_xe(sc); + u32 count = XE_SYSCTRL_EVENT_FLOOD; + size_t len; + int ret; + + do { + memset(response, 0, sizeof(*response)); + + ret = xe_sysctrl_send_command(sc, command, &len); + if (ret) { + xe_err(xe, "sysctrl: failed to get pending event %d\n", ret); + return; + } + + if (len != sizeof(*response)) { + xe_err(xe, "sysctrl: unexpected event response length %zu (expected %zu)\n", + len, sizeof(*response)); + return; + } + + if (response->event == XE_SYSCTRL_EVENT_THRESHOLD_CROSSED) + xe_ras_counter_threshold_crossed(xe, response); + else + xe_warn(xe, "sysctrl: unexpected event %#x\n", response->event); + + if (!--count) { + xe_err(xe, "sysctrl: event flooding\n"); + return; + } + + xe_dbg(xe, "sysctrl: %u events pending\n", response->count); + } while (response->count); +} + +static void event_request_prepare(struct xe_device *xe, struct xe_sysctrl_app_msg_hdr *header, + struct xe_sysctrl_event_request *request) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + + header->data = REG_FIELD_PREP(APP_HDR_GROUP_ID_MASK, XE_SYSCTRL_GROUP_GFSP) | + REG_FIELD_PREP(APP_HDR_COMMAND_MASK, XE_SYSCTRL_CMD_GET_PENDING_EVENT); + + request->vector = xe_device_has_msix(xe) ? XE_IRQ_DEFAULT_MSIX : 0; + request->fn = PCI_FUNC(pdev->devfn); +} + +/** + * xe_sysctrl_event() - Handler for System Controller events + * @sc: System Controller instance + * + * Handle events generated by System Controller. + */ +void xe_sysctrl_event(struct xe_sysctrl *sc) +{ + struct xe_sysctrl_mailbox_command command = {}; + struct xe_sysctrl_event_response response = {}; + struct xe_sysctrl_event_request request = {}; + struct xe_sysctrl_app_msg_hdr header = {}; + + xe_device_assert_mem_access(sc_to_xe(sc)); + event_request_prepare(sc_to_xe(sc), &header, &request); + + command.header = header; + command.data_in = &request; + command.data_in_len = sizeof(request); + command.data_out = &response; + command.data_out_len = sizeof(response); + + guard(mutex)(&sc->event_lock); + get_pending_event(sc, &command); +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event_types.h b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h new file mode 100644 index 000000000000..c16c66b9fa7f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_EVENT_TYPES_H_ +#define _XE_SYSCTRL_EVENT_TYPES_H_ + +#include <linux/types.h> + +#define XE_SYSCTRL_EVENT_DATA_LEN 59 + +/* Modify as needed */ +#define XE_SYSCTRL_EVENT_FLOOD 16 + +/** + * enum xe_sysctrl_event - Events reported by System Controller + * + * @XE_SYSCTRL_EVENT_THRESHOLD_CROSSED: Error counter threshold crossed + */ +enum xe_sysctrl_event { + XE_SYSCTRL_EVENT_THRESHOLD_CROSSED = 0x01, +}; + +/** + * struct xe_sysctrl_event_request - Request structure for pending event + */ +struct xe_sysctrl_event_request { + /** @vector: MSI-X vector that was triggered */ + u32 vector; + /** @fn: Function index (0-7) of PCIe device */ + u32 fn:8; + /** @reserved: Reserved for future use */ + u32 reserved:24; + /** @reserved1: Reserved for future use */ + u32 reserved1[2]; +} __packed; + +/** + * struct xe_sysctrl_event_response - Response structure for pending event + */ +struct xe_sysctrl_event_response { + /** @count: Pending event count after this response */ + u32 count; + /** @event: Pending event type */ + u32 event; + /** @timestamp: Timestamp of most recent event */ + u64 timestamp; + /** @extended: Event has extended payload */ + u32 extended:1; + /** @reserved: Reserved for future use */ + u32 reserved:31; + /** @data: Generic event data */ + u32 data[XE_SYSCTRL_EVENT_DATA_LEN]; +} __packed; + +#endif /* _XE_SYSCTRL_EVENT_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c new file mode 100644 index 000000000000..3caa9f15875f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2026 Intel Corporation + */ + +#include <linux/bitfield.h> +#include <linux/cleanup.h> +#include <linux/minmax.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "regs/xe_sysctrl_regs.h" +#include "xe_device.h" +#include "xe_mmio.h" +#include "xe_pm.h" +#include "xe_printk.h" +#include "xe_sysctrl.h" +#include "xe_sysctrl_mailbox.h" +#include "xe_sysctrl_mailbox_types.h" + +struct xe_sysctrl_mailbox_msg_hdr { + __le32 data; +} __packed; + +#define XE_SYSCTRL_HDR_GROUP_ID(hdr) \ + FIELD_GET(SYSCTRL_HDR_GROUP_ID_MASK, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_COMMAND(hdr) \ + FIELD_GET(SYSCTRL_HDR_COMMAND_MASK, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_IS_RESPONSE(hdr) \ + FIELD_GET(SYSCTRL_HDR_IS_RESPONSE, le32_to_cpu((hdr)->data)) + +#define XE_SYSCTRL_HDR_RESULT(hdr) \ + FIELD_GET(SYSCTRL_HDR_RESULT_MASK, le32_to_cpu((hdr)->data)) + +static bool sysctrl_wait_bit_clear(struct xe_sysctrl *sc, u32 bit_mask, + unsigned int timeout_ms) +{ + int ret; + + ret = xe_mmio_wait32_not(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask, + timeout_ms * 1000, NULL, false); + + return ret == 0; +} + +static bool sysctrl_wait_bit_set(struct xe_sysctrl *sc, u32 bit_mask, + unsigned int timeout_ms) +{ + int ret; + + ret = xe_mmio_wait32(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask, + timeout_ms * 1000, NULL, false); + + return ret == 0; +} + +static int sysctrl_write_frame(struct xe_sysctrl *sc, const void *frame, + size_t len) +{ + static const struct xe_reg regs[] = { + SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3 + }; + struct xe_device *xe = sc_to_xe(sc); + u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0}; + u32 dw = DIV_ROUND_UP(len, sizeof(u32)); + u32 i; + + xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE); + + memcpy(val, frame, len); + + for (i = 0; i < dw; i++) + xe_mmio_write32(sc->mmio, regs[i], val[i]); + + return 0; +} + +static int sysctrl_read_frame(struct xe_sysctrl *sc, void *frame, + size_t len) +{ + static const struct xe_reg regs[] = { + SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3 + }; + struct xe_device *xe = sc_to_xe(sc); + u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0}; + u32 dw = DIV_ROUND_UP(len, sizeof(u32)); + u32 i; + + xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE); + + for (i = 0; i < dw; i++) + val[i] = xe_mmio_read32(sc->mmio, regs[i]); + + memcpy(frame, val, len); + + return 0; +} + +static void sysctrl_clear_response(struct xe_sysctrl *sc) +{ + xe_mmio_rmw32(sc->mmio, SYSCTRL_MB_CTRL, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, 0); +} + +static int sysctrl_prepare_command(struct xe_device *xe, + u8 group_id, u8 command, + const void *data_in, size_t data_in_len, + u8 **mbox_cmd, size_t *cmd_size) +{ + struct xe_sysctrl_mailbox_msg_hdr *hdr; + size_t size; + u8 *buffer; + + xe_assert(xe, command <= SYSCTRL_HDR_COMMAND_MAX); + + if (data_in_len > XE_SYSCTRL_MB_MAX_MESSAGE_SIZE - sizeof(*hdr)) { + xe_err(xe, "sysctrl: Input data too large: %zu bytes\n", data_in_len); + return -EINVAL; + } + + size = sizeof(*hdr) + data_in_len; + + buffer = kmalloc(size, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer; + hdr->data = cpu_to_le32(FIELD_PREP(SYSCTRL_HDR_GROUP_ID_MASK, group_id) | + FIELD_PREP(SYSCTRL_HDR_COMMAND_MASK, command)); + + if (data_in && data_in_len) + memcpy(buffer + sizeof(*hdr), data_in, data_in_len); + + *mbox_cmd = buffer; + *cmd_size = size; + + return 0; +} + +static int sysctrl_send_frames(struct xe_sysctrl *sc, + const u8 *mbox_cmd, + size_t cmd_size, unsigned int timeout_ms) +{ + struct xe_device *xe = sc_to_xe(sc); + u32 ctrl_reg, total_frames, frame; + size_t bytes_sent, frame_size; + + total_frames = DIV_ROUND_UP(cmd_size, XE_SYSCTRL_MB_FRAME_SIZE); + + if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) { + xe_err(xe, "sysctrl: Mailbox busy\n"); + return -EBUSY; + } + + sc->phase_bit ^= 1; + bytes_sent = 0; + + for (frame = 0; frame < total_frames; frame++) { + frame_size = min_t(size_t, cmd_size - bytes_sent, XE_SYSCTRL_MB_FRAME_SIZE); + + if (sysctrl_write_frame(sc, mbox_cmd + bytes_sent, frame_size)) { + xe_err(xe, "sysctrl: Failed to write frame %u\n", frame); + sc->phase_bit = 0; + return -EIO; + } + + ctrl_reg = SYSCTRL_MB_CTRL_RUN_BUSY | + REG_FIELD_PREP(SYSCTRL_FRAME_CURRENT_MASK, frame) | + REG_FIELD_PREP(SYSCTRL_FRAME_TOTAL_MASK, total_frames - 1) | + SYSCTRL_MB_CTRL_CMD | + (sc->phase_bit ? SYSCTRL_FRAME_PHASE : 0); + + xe_mmio_write32(sc->mmio, SYSCTRL_MB_CTRL, ctrl_reg); + + if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) { + xe_err(xe, "sysctrl: Frame %u acknowledgment timeout\n", frame); + sc->phase_bit = 0; + return -ETIMEDOUT; + } + + bytes_sent += frame_size; + } + + return 0; +} + +static int sysctrl_process_frame(struct xe_sysctrl *sc, void *out, + size_t frame_size, unsigned int timeout_ms, + bool *done) +{ + u32 curr_frame, total_frames, ctrl_reg; + struct xe_device *xe = sc_to_xe(sc); + int ret; + + if (!sysctrl_wait_bit_set(sc, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, timeout_ms)) { + xe_err(xe, "sysctrl: Response frame timeout\n"); + return -ETIMEDOUT; + } + + ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); + total_frames = FIELD_GET(SYSCTRL_FRAME_TOTAL_MASK, ctrl_reg); + curr_frame = FIELD_GET(SYSCTRL_FRAME_CURRENT_MASK, ctrl_reg); + + ret = sysctrl_read_frame(sc, out, frame_size); + if (ret) + return ret; + + sysctrl_clear_response(sc); + + if (curr_frame == total_frames) + *done = true; + + return 0; +} + +static int sysctrl_receive_frames(struct xe_sysctrl *sc, + const struct xe_sysctrl_mailbox_msg_hdr *req, + void *data_out, size_t data_out_len, + size_t *rdata_len, unsigned int timeout_ms) +{ + struct xe_sysctrl_mailbox_msg_hdr *hdr; + struct xe_device *xe = sc_to_xe(sc); + size_t remain = sizeof(*hdr) + data_out_len; + u8 *buffer __free(kfree) = kzalloc(remain, GFP_KERNEL); + size_t frame_size; + bool done = false; + int ret = 0; + u8 *out; + + if (!buffer) + return -ENOMEM; + + out = buffer; + while (!done && remain) { + frame_size = min_t(size_t, remain, XE_SYSCTRL_MB_FRAME_SIZE); + + ret = sysctrl_process_frame(sc, out, frame_size, timeout_ms, + &done); + if (ret) + return ret; + + remain -= frame_size; + out += frame_size; + } + + hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer; + + if (!XE_SYSCTRL_HDR_IS_RESPONSE(hdr) || + XE_SYSCTRL_HDR_GROUP_ID(hdr) != XE_SYSCTRL_HDR_GROUP_ID(req) || + XE_SYSCTRL_HDR_COMMAND(hdr) != XE_SYSCTRL_HDR_COMMAND(req)) { + xe_err(xe, "sysctrl: Response header mismatch\n"); + return -EPROTO; + } + + if (XE_SYSCTRL_HDR_RESULT(hdr) != 0) { + xe_err(xe, "sysctrl: Firmware error: 0x%02lx\n", + XE_SYSCTRL_HDR_RESULT(hdr)); + return -EIO; + } + + memcpy(data_out, hdr + 1, data_out_len); + *rdata_len = out - buffer - sizeof(*hdr); + + return 0; +} + +static int sysctrl_send_command(struct xe_sysctrl *sc, + const u8 *mbox_cmd, size_t cmd_size, + void *data_out, size_t data_out_len, + size_t *rdata_len, unsigned int timeout_ms) +{ + const struct xe_sysctrl_mailbox_msg_hdr *hdr; + size_t received; + int ret; + + ret = sysctrl_send_frames(sc, mbox_cmd, cmd_size, timeout_ms); + if (ret) + return ret; + + if (!data_out || !rdata_len) + return 0; + + hdr = (const struct xe_sysctrl_mailbox_msg_hdr *)mbox_cmd; + + ret = sysctrl_receive_frames(sc, hdr, data_out, data_out_len, + &received, timeout_ms); + if (ret) + return ret; + + *rdata_len = received; + + return 0; +} + +/** + * xe_sysctrl_mailbox_init - Initialize System Controller mailbox interface + * @sc: System controller structure + * + * Initialize system controller mailbox interface for communication. + */ +void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc) +{ + u32 ctrl_reg; + + ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL); + sc->phase_bit = (ctrl_reg & SYSCTRL_FRAME_PHASE) ? 1 : 0; +} + +/** + * xe_sysctrl_send_command() - Send mailbox command to System Controller + * @sc: System Controller instance + * @cmd: Command descriptor containing request header and payload buffers + * @rdata_len: Pointer to store actual response data length + * + * Sends a mailbox command to System Controller firmware using + * System Controller mailbox and waits for a response. + * + * Request payload is provided via @cmd->data_in and @cmd->data_in_len. + * If a response is expected, @cmd->data_out must point to a buffer of + * size @cmd->data_out_len supplied by caller. + * + * On success, @rdata_len is updated with number of valid response bytes + * returned by firmware, bounded by @cmd->data_out_len. + * + * Return: 0 on success, or negative errno on failure. + */ +int xe_sysctrl_send_command(struct xe_sysctrl *sc, + struct xe_sysctrl_mailbox_command *cmd, + size_t *rdata_len) +{ + struct xe_device *xe = sc_to_xe(sc); + u8 group_id, command_code; + u8 *mbox_cmd = NULL; + size_t cmd_size = 0; + int ret; + + guard(xe_pm_runtime_noresume)(xe); + + if (!xe->info.has_sysctrl) + return -ENODEV; + + xe_assert(xe, cmd->data_in || cmd->data_out); + xe_assert(xe, !cmd->data_in || cmd->data_in_len); + xe_assert(xe, !cmd->data_out || cmd->data_out_len); + + group_id = XE_SYSCTRL_APP_HDR_GROUP_ID(&cmd->header); + command_code = XE_SYSCTRL_APP_HDR_COMMAND(&cmd->header); + + might_sleep(); + + ret = sysctrl_prepare_command(xe, group_id, command_code, + cmd->data_in, cmd->data_in_len, + &mbox_cmd, &cmd_size); + if (ret) { + xe_err(xe, "sysctrl: Failed to prepare command: %pe\n", ERR_PTR(ret)); + return ret; + } + + guard(mutex)(&sc->cmd_lock); + + ret = sysctrl_send_command(sc, mbox_cmd, cmd_size, + cmd->data_out, cmd->data_out_len, rdata_len, + XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS); + if (ret) + xe_err(xe, "sysctrl: Mailbox command failed: %pe\n", ERR_PTR(ret)); + + kfree(mbox_cmd); + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h new file mode 100644 index 000000000000..f67e9234de48 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_MAILBOX_H_ +#define _XE_SYSCTRL_MAILBOX_H_ + +#include <linux/bitfield.h> +#include <linux/types.h> + +#include "abi/xe_sysctrl_abi.h" + +struct xe_sysctrl; +struct xe_sysctrl_mailbox_command; + +#define XE_SYSCTRL_APP_HDR_GROUP_ID(hdr) \ + FIELD_GET(APP_HDR_GROUP_ID_MASK, (hdr)->data) + +#define XE_SYSCTRL_APP_HDR_COMMAND(hdr) \ + FIELD_GET(APP_HDR_COMMAND_MASK, (hdr)->data) + +#define XE_SYSCTRL_APP_HDR_VERSION(hdr) \ + FIELD_GET(APP_HDR_VERSION_MASK, (hdr)->data) + +void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc); +int xe_sysctrl_send_command(struct xe_sysctrl *sc, + struct xe_sysctrl_mailbox_command *cmd, + size_t *rdata_len); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h new file mode 100644 index 000000000000..84d7c647e743 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_MAILBOX_TYPES_H_ +#define _XE_SYSCTRL_MAILBOX_TYPES_H_ + +#include <linux/types.h> + +#include "abi/xe_sysctrl_abi.h" + +/** + * enum xe_sysctrl_group - System Controller command groups + * + * @XE_SYSCTRL_GROUP_GFSP: GFSP group + */ +enum xe_sysctrl_group { + XE_SYSCTRL_GROUP_GFSP = 0x01, +}; + +/** + * enum xe_sysctrl_gfsp_cmd - Commands supported by GFSP group + * + * @XE_SYSCTRL_CMD_GET_PENDING_EVENT: Retrieve pending event + */ +enum xe_sysctrl_gfsp_cmd { + XE_SYSCTRL_CMD_GET_PENDING_EVENT = 0x07, +}; + +/** + * struct xe_sysctrl_mailbox_command - System Controller mailbox command + */ +struct xe_sysctrl_mailbox_command { + /** @header: Application message header containing command information */ + struct xe_sysctrl_app_msg_hdr header; + + /** @data_in: Pointer to input payload data (can be NULL if no input data) */ + void *data_in; + + /** @data_in_len: Size of input payload in bytes (0 if no input data) */ + size_t data_in_len; + + /** @data_out: Pointer to output buffer for response data (can be NULL if no response) */ + void *data_out; + + /** @data_out_len: Size of output buffer in bytes (0 if no response expected) */ + size_t data_out_len; +}; + +#define XE_SYSCTRL_MB_FRAME_SIZE 16 +#define XE_SYSCTRL_MB_MAX_FRAMES 64 +#define XE_SYSCTRL_MB_MAX_MESSAGE_SIZE \ + (XE_SYSCTRL_MB_FRAME_SIZE * XE_SYSCTRL_MB_MAX_FRAMES) + +#define XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS 500 + +#endif diff --git a/drivers/gpu/drm/xe/xe_sysctrl_types.h b/drivers/gpu/drm/xe/xe_sysctrl_types.h new file mode 100644 index 000000000000..66ba24f43017 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sysctrl_types.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2026 Intel Corporation + */ + +#ifndef _XE_SYSCTRL_TYPES_H_ +#define _XE_SYSCTRL_TYPES_H_ + +#include <linux/mutex.h> +#include <linux/types.h> +#include <linux/workqueue_types.h> + +struct xe_mmio; + +/** + * struct xe_sysctrl - System Controller driver context + * + * This structure maintains the runtime state for System Controller + * communication. All fields are initialized during xe_sysctrl_init() + * and protected appropriately for concurrent access. + */ +struct xe_sysctrl { + /** @mmio: MMIO region for system control registers */ + struct xe_mmio *mmio; + + /** @cmd_lock: Mutex protecting mailbox command operations */ + struct mutex cmd_lock; + + /** @phase_bit: Message boundary phase toggle bit (0 or 1) */ + bool phase_bit; + + /** @work: Pending events worker */ + struct work_struct work; + + /** @event_lock: Mutex protecting pending events */ + struct mutex event_lock; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_tile_types.h b/drivers/gpu/drm/xe/xe_tile_types.h index 33932fd547d7..0048100ccb72 100644 --- a/drivers/gpu/drm/xe/xe_tile_types.h +++ b/drivers/gpu/drm/xe/xe_tile_types.h @@ -106,8 +106,6 @@ struct xe_tile { struct xe_lmtt lmtt; } pf; struct { - /** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */ - struct xe_ggtt_node *ggtt_balloon[2]; /** @sriov.vf.self_config: VF configuration data */ struct xe_tile_sriov_vf_selfconfig self_config; } vf; diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c index 10dcd4abb00f..bbd21d393062 100644 --- a/drivers/gpu/drm/xe/xe_tlb_inval.c +++ b/drivers/gpu/drm/xe/xe_tlb_inval.c @@ -529,7 +529,7 @@ int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid, struct xe_tile *tile; u32 fence_id = 0; u8 id; - int err; + int err = 0; batch->num_fences = 0; if (!tile_mask) diff --git a/drivers/gpu/drm/xe/xe_trace_lrc.h b/drivers/gpu/drm/xe/xe_trace_lrc.h index d525cbee1e34..5c4cfa0c1fe9 100644 --- a/drivers/gpu/drm/xe/xe_trace_lrc.h +++ b/drivers/gpu/drm/xe/xe_trace_lrc.h @@ -12,6 +12,7 @@ #include <linux/tracepoint.h> #include <linux/types.h> +#include "xe_exec_queue_types.h" #include "xe_gt_types.h" #include "xe_lrc.h" #include "xe_lrc_types.h" @@ -42,6 +43,32 @@ TRACE_EVENT(xe_lrc_update_timestamp, __get_str(device_id)) ); +TRACE_EVENT(xe_lrc_update_queue_timestamp, + TP_PROTO(struct xe_lrc *lrc, uint64_t old), + TP_ARGS(lrc, old), + TP_STRUCT__entry( + __field(struct xe_lrc *, lrc) + __field(struct xe_lrc *, primary_lrc) + __field(u64, old) + __field(u64, new) + __string(name, lrc->fence_ctx.name) + __string(device_id, __dev_name_lrc(lrc)) + ), + + TP_fast_assign( + __entry->lrc = lrc; + __entry->primary_lrc = lrc->multi_queue.primary_lrc; + __entry->old = old; + __entry->new = lrc->queue_timestamp; + __assign_str(name); + __assign_str(device_id); + ), + TP_printk("lrc=%p primary_lrc=%p lrc->name=%s old=%llu new=%llu device_id:%s", + __entry->lrc, __entry->primary_lrc, __get_str(name), + __entry->old, __entry->new, + __get_str(device_id)) +); + #endif /* This part must be outside protection */ diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c index 27c9d72222cf..5e9070739e65 100644 --- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c @@ -19,30 +19,11 @@ #include "xe_device.h" #include "xe_gt_printk.h" #include "xe_mmio.h" -#include "xe_res_cursor.h" #include "xe_sriov.h" #include "xe_ttm_stolen_mgr.h" -#include "xe_ttm_vram_mgr.h" #include "xe_vram.h" #include "xe_wa.h" -struct xe_ttm_stolen_mgr { - struct xe_ttm_vram_mgr base; - - /* PCI base offset */ - resource_size_t io_base; - /* GPU base offset */ - resource_size_t stolen_base; - - void __iomem *mapping; -}; - -static inline struct xe_ttm_stolen_mgr * -to_stolen_mgr(struct ttm_resource_manager *man) -{ - return container_of(man, struct xe_ttm_stolen_mgr, base.manager); -} - /** * xe_ttm_stolen_cpu_access_needs_ggtt() - If we can't directly CPU access * stolen, can we then fallback to mapping through the GGTT. @@ -210,12 +191,19 @@ static u64 detect_stolen(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr) #endif } +static void xe_ttm_stolen_mgr_fini(struct drm_device *dev, void *arg) +{ + struct xe_device *xe = to_xe_device(dev); + + ttm_range_man_fini_nocheck(&xe->ttm, XE_PL_STOLEN); +} + int xe_ttm_stolen_mgr_init(struct xe_device *xe) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); struct xe_ttm_stolen_mgr *mgr; u64 stolen_size, io_size; - int err; + int ret; mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL); if (!mgr) @@ -244,12 +232,12 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe) if (mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe)) io_size = stolen_size; - err = __xe_ttm_vram_mgr_init(xe, &mgr->base, XE_PL_STOLEN, stolen_size, - io_size, PAGE_SIZE); - if (err) { - drm_dbg_kms(&xe->drm, "Stolen mgr init failed: %i\n", err); - return err; - } + ret = ttm_range_man_init_nocheck(&xe->ttm, XE_PL_STOLEN, false, + stolen_size >> PAGE_SHIFT); + if (ret) + return ret; + + xe->mem.stolen_mgr = mgr; drm_dbg_kms(&xe->drm, "Initialized stolen memory support with %llu bytes\n", stolen_size); @@ -257,36 +245,32 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe) if (io_size) mgr->mapping = devm_ioremap_wc(&pdev->dev, mgr->io_base, io_size); - return 0; + return drmm_add_action_or_reset(&xe->drm, xe_ttm_stolen_mgr_fini, mgr); } u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset) { struct xe_device *xe = xe_bo_device(bo); - struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); - struct xe_ttm_stolen_mgr *mgr = to_stolen_mgr(ttm_mgr); - struct xe_res_cursor cur; + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; XE_WARN_ON(!mgr->io_base); if (xe_ttm_stolen_cpu_access_needs_ggtt(xe)) return mgr->io_base + xe_bo_ggtt_addr(bo) + offset; - xe_res_first(bo->ttm.resource, offset, 4096, &cur); - return mgr->io_base + cur.start; + /* Range allocator: res->start is in pages. */ + return mgr->io_base + (bo->ttm.resource->start << PAGE_SHIFT) + offset; } static int __xe_ttm_stolen_io_mem_reserve_bar2(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr, struct ttm_resource *mem) { - struct xe_res_cursor cur; - if (!mgr->io_base) return -EIO; - xe_res_first(mem, 0, 4096, &cur); - mem->bus.offset = cur.start; + /* Range allocator always produces contiguous allocations. */ + mem->bus.offset = mem->start << PAGE_SHIFT; drm_WARN_ON(&xe->drm, !(mem->placement & TTM_PL_FLAG_CONTIGUOUS)); @@ -329,8 +313,7 @@ static int __xe_ttm_stolen_io_mem_reserve_stolen(struct xe_device *xe, int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem) { - struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN); - struct xe_ttm_stolen_mgr *mgr = ttm_mgr ? to_stolen_mgr(ttm_mgr) : NULL; + struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr; if (!mgr || !mgr->io_base) return -EIO; @@ -343,8 +326,5 @@ int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem) u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe) { - struct xe_ttm_stolen_mgr *mgr = - to_stolen_mgr(ttm_manager_type(&xe->ttm, XE_PL_STOLEN)); - - return mgr->stolen_base; + return xe->mem.stolen_mgr->stolen_base; } diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h index 8e877d1e839b..0675106d535b 100644 --- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h +++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h @@ -12,6 +12,18 @@ struct ttm_resource; struct xe_bo; struct xe_device; +/** + * struct xe_ttm_stolen_mgr - Xe TTM stolen memory manager + */ +struct xe_ttm_stolen_mgr { + /** @io_base: PCI base offset for CPU I/O access */ + resource_size_t io_base; + /** @stolen_base: GPU base offset */ + resource_size_t stolen_base; + /** @mapping: I/O memory mapping for CPU access */ + void __iomem *mapping; +}; + int xe_ttm_stolen_mgr_init(struct xe_device *xe); int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem); bool xe_ttm_stolen_cpu_access_needs_ggtt(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 5fd0d5506a7e..b518f7dec680 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -292,8 +292,6 @@ static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg) ttm_resource_manager_cleanup(&mgr->manager); ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL); - - mutex_destroy(&mgr->lock); } int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, @@ -301,18 +299,19 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, u64 default_page_size) { struct ttm_resource_manager *man = &mgr->manager; + const char *name; int err; - if (mem_type != XE_PL_STOLEN) { - const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; - man->cg = drmm_cgroup_register_region(&xe->drm, name, size); - if (IS_ERR(man->cg)) - return PTR_ERR(man->cg); - } + name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; + man->cg = drmm_cgroup_register_region(&xe->drm, name, size); + if (IS_ERR(man->cg)) + return PTR_ERR(man->cg); man->func = &xe_ttm_vram_mgr_func; mgr->mem_type = mem_type; - mutex_init(&mgr->lock); + err = drmm_mutex_init(&xe->drm, &mgr->lock); + if (err) + return err; mgr->default_page_size = default_page_size; mgr->visible_size = io_size; mgr->visible_avail = io_size; @@ -322,6 +321,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, if (err) return err; + gpu_buddy_driver_set_lock(&mgr->mm, &mgr->lock); ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager); ttm_resource_manager_set_used(&mgr->manager, true); diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c index f8de6a4bf189..9a1b3862e192 100644 --- a/drivers/gpu/drm/xe/xe_tuning.c +++ b/drivers/gpu/drm/xe/xe_tuning.c @@ -43,7 +43,7 @@ static const struct xe_rtp_entry_sr gt_tunings[] = { REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f))) }, { XE_RTP_NAME("Tuning: Compression Overfetch"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3499), FUNC(xe_rtp_match_has_flat_ccs)), XE_RTP_ACTIONS(CLR(CCCHKNREG1, ENCOMPPERFFIX), SET(CCCHKNREG1, L3CMPCTRL)) @@ -97,7 +97,7 @@ static const struct xe_rtp_entry_sr gt_tunings[] = { { XE_RTP_NAME("Tuning: Set STLB Bank Hash Mode to 4KB"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED), IS_INTEGRATED), - XE_RTP_ACTIONS(FIELD_SET(XEHP_GAMSTLB_CTRL, BANK_HASH_MODE, + XE_RTP_ACTIONS(FIELD_SET(GAMSTLB_CTRL, BANK_HASH_MODE, BANK_HASH_4KB_MODE)) }, }; @@ -124,12 +124,17 @@ static const struct xe_rtp_entry_sr engine_tunings[] = { GHWSP_CSB_REPORT_DIS, XE_RTP_ACTION_FLAG(ENGINE_BASE))) }, + { XE_RTP_NAME("Tuning: TileY 2x2 Walk"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN2, TILEY_LOCALID)) + }, }; static const struct xe_rtp_entry_sr lrc_tunings[] = { { XE_RTP_NAME("Tuning: Windower HW Filtering"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3599), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, HW_FILTERING)) + XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, HW_FILTERING)) }, /* DG2 */ diff --git a/drivers/gpu/drm/xe/xe_tuning.h b/drivers/gpu/drm/xe/xe_tuning.h index c1cc5927fda7..d18e187debf6 100644 --- a/drivers/gpu/drm/xe/xe_tuning.h +++ b/drivers/gpu/drm/xe/xe_tuning.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_TUNING_ -#define _XE_TUNING_ +#ifndef _XE_TUNING_H_ +#define _XE_TUNING_H_ struct drm_printer; struct xe_gt; diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c index 9cebb2490245..df2aa196f6f9 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.c +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -214,6 +214,17 @@ static struct xe_device *uc_fw_to_xe(struct xe_uc_fw *uc_fw) return gt_to_xe(uc_fw_to_gt(uc_fw)); } +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) +void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status) +{ + xe_gt_dbg(uc_fw_to_gt(uc_fw), "%s %s->%s\n", + xe_uc_fw_type_repr(uc_fw->type), + xe_uc_fw_status_repr(uc_fw->status), + xe_uc_fw_status_repr(status)); + uc_fw->__status = status; +} +#endif + static void uc_fw_auto_select(struct xe_device *xe, struct xe_uc_fw *uc_fw) { diff --git a/drivers/gpu/drm/xe/xe_uc_fw.h b/drivers/gpu/drm/xe/xe_uc_fw.h index 6195e353f269..bb281b72a677 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.h +++ b/drivers/gpu/drm/xe/xe_uc_fw.h @@ -25,11 +25,15 @@ static inline u32 xe_uc_fw_rsa_offset(struct xe_uc_fw *uc_fw) return sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->css_offset; } +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) +void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status); +#else static inline void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status) { uc_fw->__status = status; } +#endif static inline const char *xe_uc_fw_status_repr(enum xe_uc_fw_status status) diff --git a/drivers/gpu/drm/xe/xe_uc_fw_abi.h b/drivers/gpu/drm/xe/xe_uc_fw_abi.h index 3c9a63d13032..74b888904fdc 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw_abi.h +++ b/drivers/gpu/drm/xe/xe_uc_fw_abi.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_UC_FW_ABI_H -#define _XE_UC_FW_ABI_H +#ifndef _XE_UC_FW_ABI_H_ +#define _XE_UC_FW_ABI_H_ #include <linux/build_bug.h> #include <linux/types.h> diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 56e2db50bb36..b01f31ed4417 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -634,9 +634,9 @@ void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf) e->address_precision = SZ_4K; e->access_type = pf->consumer.access_type; e->fault_type = FIELD_GET(XE_PAGEFAULT_TYPE_MASK, - pf->consumer.fault_type_level), + pf->consumer.fault_type_level); e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK, - pf->consumer.fault_type_level), + pf->consumer.fault_type_level); list_add_tail(&e->list, &vm->faults.list); vm->faults.len++; @@ -1120,6 +1120,25 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, xe_bo_assert_held(bo); + /* + * Reject only WILLNEED mappings on DONTNEED/PURGED BOs. This + * gates new vm_bind ioctls (user supplies WILLNEED) while + * still allowing partial-unbind / remap splits whose new VMAs + * inherit the parent's DONTNEED attr. It must also run before + * xe_bo_willneed_get_locked() below so a 0->1 holder bump + * cannot silently promote DONTNEED back to WILLNEED. + */ + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) { + if (xe_bo_madv_is_dontneed(bo)) { + xe_vma_free(vma); + return ERR_PTR(-EBUSY); + } + if (xe_bo_is_purged(bo)) { + xe_vma_free(vma); + return ERR_PTR(-EINVAL); + } + } + vm_bo = drm_gpuvm_bo_obtain_locked(vma->gpuva.vm, &bo->ttm.base); if (IS_ERR(vm_bo)) { xe_vma_free(vma); @@ -1131,6 +1150,10 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, vma->gpuva.gem.offset = bo_offset_or_userptr; drm_gpuva_link(&vma->gpuva, vm_bo); drm_gpuvm_bo_put(vm_bo); + + xe_bo_vma_count_inc_locked(bo); + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) + xe_bo_willneed_get_locked(bo); } else /* userptr or null */ { if (!is_null && !is_cpu_addr_mirror) { struct xe_userptr_vma *uvma = to_userptr_vma(vma); @@ -1208,7 +1231,10 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) xe_bo_assert_held(bo); drm_gpuva_unlink(&vma->gpuva); - xe_bo_recompute_purgeable_state(bo); + + xe_bo_vma_count_dec_locked(bo); + if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) + xe_bo_willneed_put_locked(bo); } xe_vm_assert_held(vm); @@ -1399,9 +1425,9 @@ static u16 pde_pat_index(struct xe_bo *bo) * something which is always safe). */ if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached) - pat_index = xe->pat.idx[XE_CACHE_WB]; + pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB); else - pat_index = xe->pat.idx[XE_CACHE_NONE]; + pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE); xe_assert(xe, pat_index <= 3); @@ -3016,7 +3042,7 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, * @res_evict: Allow evicting resources during validation * @validate: Perform BO validation * @request_decompress: Request BO decompression - * @check_purged: Reject operation if BO is purged + * @check_purged: Reject operation if BO is DONTNEED or PURGED */ struct xe_vma_lock_and_validate_flags { u32 res_evict : 1; @@ -3030,6 +3056,7 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, { struct xe_bo *bo = xe_vma_bo(vma); struct xe_vm *vm = xe_vma_vm(vma); + bool validate_bo = flags.validate; int err = 0; if (bo) { @@ -3044,7 +3071,11 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, err = -EINVAL; /* BO already purged */ } - if (!err && flags.validate) + /* Don't validate the BO for DONTNEED/PURGED remap remnants. */ + if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_WILLNEED) + validate_bo = false; + + if (!err && validate_bo) err = xe_bo_validate(bo, vm, xe_vm_allow_vm_eviction(vm) && flags.res_evict, exec); @@ -3152,7 +3183,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, op->map.immediate, .request_decompress = op->map.request_decompress, - .check_purged = true, + .check_purged = false, }); break; case DRM_GPUVA_OP_REMAP: @@ -3174,7 +3205,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, .res_evict = res_evict, .validate = true, .request_decompress = false, - .check_purged = true, + .check_purged = false, }); if (!err && op->remap.next) err = vma_lock_and_validate(exec, op->remap.next, @@ -3182,7 +3213,7 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, .res_evict = res_evict, .validate = true, .request_decompress = false, - .check_purged = true, + .check_purged = false, }); break; case DRM_GPUVA_OP_UNMAP: @@ -3211,9 +3242,11 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, } /* - * Prefetch attempts to migrate BO's backing store without - * repopulating it first. Purged BOs have no backing store - * to migrate, so reject the operation. + * PREFETCH is the only op that still gates on BO purge state. + * MAP/REMAP handle this inside xe_vma_create() so partial + * unbind on a DONTNEED BO still works. PREFETCH skips + * xe_vma_create() and would migrate a BO with no backing + * store, so reject DONTNEED/PURGED here. */ err = vma_lock_and_validate(exec, gpuva_to_vma(op->base.prefetch.va), @@ -3414,7 +3447,7 @@ collect_fences: xe_assert(vm->xe, current_fence == n_fence); dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1), - 1, false); + 1); fence = &cf->base; for_each_tile(tile, vm->xe, id) { @@ -3658,6 +3691,8 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || + XE_IOCTL_DBG(xe, !IS_DGFX(xe) && coh_mode == XE_COH_NONE && + is_cpu_addr_mirror) || XE_IOCTL_DBG(xe, xe_device_is_l2_flush_optimized(xe) && (op == DRM_XE_VM_BIND_OP_MAP_USERPTR || is_cpu_addr_mirror) && @@ -4156,7 +4191,8 @@ int xe_vm_get_property_ioctl(struct drm_device *drm, void *data, int ret = 0; if (XE_IOCTL_DBG(xe, (args->reserved[0] || args->reserved[1] || - args->reserved[2]))) + args->reserved[2] || args->extensions || + args->pad))) return -EINVAL; vm = xe_vm_lookup(xef, args->vm_id); @@ -4203,7 +4239,7 @@ struct dma_fence *xe_vm_bind_kernel_bo(struct xe_vm *vm, struct xe_bo *bo, ops = vm_bind_ioctl_ops_create(vm, &vops, bo, 0, addr, xe_bo_size(bo), DRM_XE_VM_BIND_OP_MAP, 0, 0, - vm->xe->pat.idx[cache_lvl]); + xe_cache_pat_idx(vm->xe, cache_lvl)); if (IS_ERR(ops)) { err = PTR_ERR(ops); goto release_vm_lock; @@ -4406,7 +4442,7 @@ struct xe_vm_snapshot { #define XE_VM_SNAP_FLAG_IS_NULL BIT(2) unsigned long flags; int uapi_mem_region; - int pat_index; + u16 pat_index; int cpu_caching; struct xe_bo *bo; void *data; diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index 66f00d3f5c07..c4fb29004195 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -186,147 +186,6 @@ static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, } /** - * xe_bo_is_dmabuf_shared() - Check if BO is shared via dma-buf - * @bo: Buffer object - * - * Prevent marking imported or exported dma-bufs as purgeable. - * For imported BOs, Xe doesn't own the backing store and cannot - * safely reclaim pages (exporter or other devices may still be - * using them). For exported BOs, external devices may have active - * mappings we cannot track. - * - * Return: true if BO is imported or exported, false otherwise - */ -static bool xe_bo_is_dmabuf_shared(struct xe_bo *bo) -{ - struct drm_gem_object *obj = &bo->ttm.base; - - /* Imported: exporter owns backing store */ - if (drm_gem_is_imported(obj)) - return true; - - /* Exported: external devices may be accessing */ - if (obj->dma_buf) - return true; - - return false; -} - -/** - * enum xe_bo_vmas_purge_state - VMA purgeable state aggregation - * - * Distinguishes whether a BO's VMAs are all DONTNEED, have at least - * one WILLNEED, or have no VMAs at all. - * - * Enum values align with XE_MADV_PURGEABLE_* states for consistency. - */ -enum xe_bo_vmas_purge_state { - /** @XE_BO_VMAS_STATE_WILLNEED: At least one VMA is WILLNEED */ - XE_BO_VMAS_STATE_WILLNEED = 0, - /** @XE_BO_VMAS_STATE_DONTNEED: All VMAs are DONTNEED */ - XE_BO_VMAS_STATE_DONTNEED = 1, - /** @XE_BO_VMAS_STATE_NO_VMAS: BO has no VMAs */ - XE_BO_VMAS_STATE_NO_VMAS = 2, -}; - -/* - * xe_bo_recompute_purgeable_state() casts between xe_bo_vmas_purge_state and - * xe_madv_purgeable_state. Enforce that WILLNEED=0 and DONTNEED=1 match across - * both enums so the single-line cast is always valid. - */ -static_assert(XE_BO_VMAS_STATE_WILLNEED == (int)XE_MADV_PURGEABLE_WILLNEED, - "VMA purge state WILLNEED must equal madv purgeable WILLNEED"); -static_assert(XE_BO_VMAS_STATE_DONTNEED == (int)XE_MADV_PURGEABLE_DONTNEED, - "VMA purge state DONTNEED must equal madv purgeable DONTNEED"); - -/** - * xe_bo_all_vmas_dontneed() - Determine BO VMA purgeable state - * @bo: Buffer object - * - * Check all VMAs across all VMs to determine aggregate purgeable state. - * Shared BOs require unanimous DONTNEED state from all mappings. - * - * Caller must hold BO dma-resv lock. - * - * Return: XE_BO_VMAS_STATE_DONTNEED if all VMAs are DONTNEED, - * XE_BO_VMAS_STATE_WILLNEED if at least one VMA is not DONTNEED, - * XE_BO_VMAS_STATE_NO_VMAS if BO has no VMAs - */ -static enum xe_bo_vmas_purge_state xe_bo_all_vmas_dontneed(struct xe_bo *bo) -{ - struct drm_gpuvm_bo *vm_bo; - struct drm_gpuva *gpuva; - struct drm_gem_object *obj = &bo->ttm.base; - bool has_vmas = false; - - xe_bo_assert_held(bo); - - /* Shared dma-bufs cannot be purgeable */ - if (xe_bo_is_dmabuf_shared(bo)) - return XE_BO_VMAS_STATE_WILLNEED; - - drm_gem_for_each_gpuvm_bo(vm_bo, obj) { - drm_gpuvm_bo_for_each_va(gpuva, vm_bo) { - struct xe_vma *vma = gpuva_to_vma(gpuva); - - has_vmas = true; - - /* Any non-DONTNEED VMA prevents purging */ - if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_DONTNEED) - return XE_BO_VMAS_STATE_WILLNEED; - } - } - - /* - * No VMAs => preserve existing BO purgeable state. - * Avoids incorrectly flipping DONTNEED -> WILLNEED when last VMA unmapped. - */ - if (!has_vmas) - return XE_BO_VMAS_STATE_NO_VMAS; - - return XE_BO_VMAS_STATE_DONTNEED; -} - -/** - * xe_bo_recompute_purgeable_state() - Recompute BO purgeable state from VMAs - * @bo: Buffer object - * - * Walk all VMAs to determine if BO should be purgeable or not. - * Shared BOs require unanimous DONTNEED state from all mappings. - * If the BO has no VMAs the existing state is preserved. - * - * Locking: Caller must hold BO dma-resv lock. When iterating GPUVM lists, - * VM lock must also be held (write) to prevent concurrent VMA modifications. - * This is satisfied at both call sites: - * - xe_vma_destroy(): holds vm->lock write - * - madvise_purgeable(): holds vm->lock write (from madvise ioctl path) - * - * Return: nothing - */ -void xe_bo_recompute_purgeable_state(struct xe_bo *bo) -{ - enum xe_bo_vmas_purge_state vma_state; - - if (!bo) - return; - - xe_bo_assert_held(bo); - - /* - * Once purged, always purged. Cannot transition back to WILLNEED. - * This matches i915 semantics where purged BOs are permanently invalid. - */ - if (bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED) - return; - - vma_state = xe_bo_all_vmas_dontneed(bo); - - if (vma_state != (enum xe_bo_vmas_purge_state)bo->madv_purgeable && - vma_state != XE_BO_VMAS_STATE_NO_VMAS) - xe_bo_set_purgeable_state(bo, (enum xe_madv_purgeable_state)vma_state); -} - -/** * madvise_purgeable - Handle purgeable buffer object advice * @xe: XE device * @vm: VM @@ -359,12 +218,6 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm, /* BO must be locked before modifying madv state */ xe_bo_assert_held(bo); - /* Skip shared dma-bufs - no PTEs to zap */ - if (xe_bo_is_dmabuf_shared(bo)) { - vmas[i]->skip_invalidation = true; - continue; - } - /* * Once purged, always purged. Cannot transition back to WILLNEED. * This matches i915 semantics where purged BOs are permanently invalid. @@ -377,13 +230,14 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm, switch (op->purge_state_val.val) { case DRM_XE_VMA_PURGEABLE_STATE_WILLNEED: - vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED; vmas[i]->skip_invalidation = true; - - xe_bo_recompute_purgeable_state(bo); + /* Only act on a real DONTNEED -> WILLNEED transition. */ + if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_DONTNEED) { + vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED; + xe_bo_willneed_get_locked(bo); + } break; case DRM_XE_VMA_PURGEABLE_STATE_DONTNEED: - vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED; /* * Don't zap PTEs at DONTNEED time -- pages are still * alive. The zap happens in xe_bo_move_notify() right @@ -391,7 +245,11 @@ static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm, */ vmas[i]->skip_invalidation = true; - xe_bo_recompute_purgeable_state(bo); + /* Only act on a real WILLNEED -> DONTNEED transition. */ + if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) { + vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED; + xe_bo_willneed_put_locked(bo); + } break; default: /* Should never hit - values validated in madvise_args_are_sane() */ @@ -621,6 +479,45 @@ static int xe_madvise_purgeable_retained_to_user(const struct xe_madvise_details return 0; } +static bool check_pat_args_are_sane(struct xe_device *xe, + struct xe_vmas_in_madvise_range *madvise_range, + u16 pat_index) +{ + u16 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + int i; + + /* + * Using coh_none with CPU cached buffers is not allowed on iGPU. + * On iGPU the GPU shares the LLC with the CPU, so with coh_none + * the GPU bypasses CPU caches and reads directly from DRAM, + * potentially seeing stale sensitive data from previously freed + * pages. On dGPU this restriction does not apply, because the + * platform does not provide a non-coherent system memory access + * path that would violate the DMA coherency contract. + */ + if (coh_mode != XE_COH_NONE || IS_DGFX(xe)) + return true; + + for (i = 0; i < madvise_range->num_vmas; i++) { + struct xe_vma *vma = madvise_range->vmas[i]; + struct xe_bo *bo = xe_vma_bo(vma); + + if (bo) { + /* BO with WB caching + COH_NONE is not allowed */ + if (XE_IOCTL_DBG(xe, bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) + return false; + /* Imported dma-buf without caching info, assume cached */ + if (XE_IOCTL_DBG(xe, !bo->cpu_caching)) + return false; + } else if (XE_IOCTL_DBG(xe, xe_vma_is_cpu_addr_mirror(vma) || + xe_vma_is_userptr(vma))) + /* System memory (userptr/SVM) is always CPU cached */ + return false; + } + + return true; +} + static bool check_bo_args_are_sane(struct xe_vm *vm, struct xe_vma **vmas, int num_vmas, u32 atomic_val) { @@ -750,6 +647,14 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil } } + if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) { + if (!check_pat_args_are_sane(xe, &madvise_range, + args->pat_index.val)) { + err = -EINVAL; + goto free_vmas; + } + } + if (madvise_range.has_bo_vmas) { if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) { if (!check_bo_args_are_sane(vm, madvise_range.vmas, diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.h b/drivers/gpu/drm/xe/xe_vm_madvise.h index 39acd2689ca0..a3078f634c7e 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.h +++ b/drivers/gpu/drm/xe/xe_vm_madvise.h @@ -13,6 +13,4 @@ struct xe_bo; int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file); -void xe_bo_recompute_purgeable_state(struct xe_bo *bo); - #endif diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index a94827d7fbec..635ed29b9a69 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -412,10 +412,11 @@ struct xe_vm { struct xe_vma_op_map { /** @vma: VMA to map */ struct xe_vma *vma; + /** @vma_flags: VMA flags for this operation */ unsigned int vma_flags; /** @immediate: Immediate bind */ bool immediate; - /** @read_only: Read only */ + /** @invalidate_on_bind: Invalidate on bind */ bool invalidate_on_bind; /** @request_decompress: schedule decompression for GPU map */ bool request_decompress; diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index 0538dcb8b18c..23eb7edbdd57 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -13,6 +13,7 @@ #include "regs/xe_gt_regs.h" #include "regs/xe_regs.h" #include "xe_assert.h" +#include "xe_bo.h" #include "xe_device.h" #include "xe_force_wake.h" #include "xe_gt_mcr.h" @@ -250,6 +251,27 @@ static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram, } /** + * xe_map_resource_to_region - Map ttm resource to vram memory region + * @res: The ttm resource + * + * Get vram memory region using vram memory manager managing this resource + * + * Returns: pointer to xe_vram_region + */ +struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res) +{ + struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); + struct ttm_resource_manager *mgr; + struct xe_ttm_vram_mgr *vram_mgr; + + xe_assert(xe, mem_type_is_vram(res->mem_type)); + mgr = ttm_manager_type(&xe->ttm, res->mem_type); + vram_mgr = to_xe_ttm_vram_mgr(mgr); + + return container_of(vram_mgr, struct xe_vram_region, ttm); +} + +/** * xe_vram_probe() - Probe VRAM configuration * @xe: the &xe_device * diff --git a/drivers/gpu/drm/xe/xe_vram.h b/drivers/gpu/drm/xe/xe_vram.h index 72860f714fc6..dd1c8bf17922 100644 --- a/drivers/gpu/drm/xe/xe_vram.h +++ b/drivers/gpu/drm/xe/xe_vram.h @@ -10,7 +10,9 @@ struct xe_device; struct xe_vram_region; +struct ttm_resource; +struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res); int xe_vram_probe(struct xe_device *xe); struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement); diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index 546296f0220b..cb811f8a7781 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -601,6 +601,19 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(ROW_CHICKEN5, CPSS_AWARE_DIS)) }, + + /* Xe3p_XPC */ + + { XE_RTP_NAME("14026999295"), + XE_RTP_RULES(GRAPHICS_VERSION(3511), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(ROW_CHICKEN3, DIS_EU_GRF_POISON_TO_LSC)) + }, + { XE_RTP_NAME("18044193044"), + XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0), + FUNC(xe_rtp_match_first_render_or_compute)), + XE_RTP_ACTIONS(SET(TDL_CHICKEN, BIT_APQ_OPT_DIS)) + }, }; static const struct xe_rtp_entry_sr lrc_was[] = { @@ -651,7 +664,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = { }, { XE_RTP_NAME("18033852989"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST)) + XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST)) }, { XE_RTP_NAME("15016589081"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)), @@ -743,14 +756,6 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) }, - { XE_RTP_NAME("14019988906"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) - }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, { XE_RTP_NAME("14021490052"), XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(FF_MODE, @@ -762,7 +767,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = { }, { XE_RTP_NAME("22021007897"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE)) + XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE)) }, /* Xe3_LPG */ @@ -778,7 +783,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = { }, { XE_RTP_NAME("22021007897"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE)) + XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE)) }, { XE_RTP_NAME("14024681466"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)), diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h index 8fd6a5af0910..a5f7d33c1b32 100644 --- a/drivers/gpu/drm/xe/xe_wa.h +++ b/drivers/gpu/drm/xe/xe_wa.h @@ -3,8 +3,8 @@ * Copyright © 2022 Intel Corporation */ -#ifndef _XE_WA_ -#define _XE_WA_ +#ifndef _XE_WA_H_ +#define _XE_WA_H_ #include "xe_assert.h" diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules index 80b54b195f20..f8a185103b80 100644 --- a/drivers/gpu/drm/xe/xe_wa_oob.rules +++ b/drivers/gpu/drm/xe/xe_wa_oob.rules @@ -51,6 +51,13 @@ MEDIA_VERSION_RANGE(2000, 3002), FUNC(xe_rtp_match_psmi_enabled) 16023683509 MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled) MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), FUNC(xe_rtp_match_psmi_enabled) +14025515070 GRAPHICS_VERSION(2004) + MEDIA_VERSION_RANGE(1301, 3000) + MEDIA_VERSION(3002) + GRAPHICS_VERSION_RANGE(3000, 3001) + GRAPHICS_VERSION_RANGE(3003, 3005) + MEDIA_VERSION(3500) + GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0) 15015404425_disable PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER) 16026007364 MEDIA_VERSION(3000) |