diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu')
155 files changed, 5368 insertions, 2466 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig index 7fb0b93bc1ca..12e4a41bf1f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/Kconfig +++ b/drivers/gpu/drm/amd/amdgpu/Kconfig @@ -3,7 +3,6 @@ config DRM_AMDGPU tristate "AMD GPU" depends on DRM && PCI - depends on !UML select FW_LOADER select DRM_CLIENT select DRM_CLIENT_SELECTION diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index db66c6372199..ba80542ead9d 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -69,7 +69,7 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \ amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \ amdgpu_fw_attestation.o amdgpu_securedisplay.o \ - amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o \ + amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \ amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_aca.o amdgpu_dev_coredump.o \ amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o @@ -90,7 +90,7 @@ amdgpu-y += \ nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o soc24.o \ sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o \ nbio_v7_9.o aqua_vanjaram.o nbio_v7_11.o lsdma_v7_0.o hdp_v7_0.o nbif_v6_3_1.o \ - cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o + cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o nbio_v6_3_2.o # add DF block amdgpu-y += \ @@ -234,7 +234,8 @@ amdgpu-y += \ # add VPE block amdgpu-y += \ amdgpu_vpe.o \ - vpe_v6_1.o + vpe_v6_1.o \ + vpe_v2_0.o # add UMSCH block amdgpu-y += \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index fd50da4c7b18..7b09410d6d8f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -105,6 +105,7 @@ #include "amdgpu_mca.h" #include "amdgpu_aca.h" #include "amdgpu_ras.h" +#include "amdgpu_lockdep.h" #include "amdgpu_cper.h" #include "amdgpu_xcp.h" #include "amdgpu_seq64.h" @@ -267,6 +268,7 @@ extern int amdgpu_rebar; extern int amdgpu_wbrf; extern int amdgpu_user_queue; +extern int amdgpu_ptl; extern uint amdgpu_hdmi_hpd_debounce_delay_ms; @@ -464,7 +466,7 @@ int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv); #define AMDGPU_MAX_WB 1024 /* Reserve at most 1024 WB slots for amdgpu-owned rings. */ /** - * amdgpu_wb - This struct is used for small GPU memory allocation. + * struct amdgpu_wb - This struct is used for small GPU memory allocation. * * This struct is used to allocate a small amount of GPU memory that can be * used to shadow certain states into the memory. This is especially useful for @@ -538,44 +540,6 @@ struct amdgpu_allowed_register_entry { bool grbm_indexed; }; -/** - * enum amd_reset_method - Methods for resetting AMD GPU devices - * - * @AMD_RESET_METHOD_NONE: The device will not be reset. - * @AMD_RESET_LEGACY: Method reserved for SI, CIK and VI ASICs. - * @AMD_RESET_MODE0: Reset the entire ASIC. Not currently available for the - * any device. - * @AMD_RESET_MODE1: Resets all IP blocks on the ASIC (SDMA, GFX, VCN, etc.) - * individually. Suitable only for some discrete GPU, not - * available for all ASICs. - * @AMD_RESET_MODE2: Resets a lesser level of IPs compared to MODE1. Which IPs - * are reset depends on the ASIC. Notably doesn't reset IPs - * shared with the CPU on APUs or the memory controllers (so - * VRAM is not lost). Not available on all ASICs. - * @AMD_RESET_LINK: Triggers SW-UP link reset on other GPUs - * @AMD_RESET_BACO: BACO (Bus Alive, Chip Off) method powers off and on the card - * but without powering off the PCI bus. Suitable only for - * discrete GPUs. - * @AMD_RESET_PCI: Does a full bus reset using core Linux subsystem PCI reset - * and does a secondary bus reset or FLR, depending on what the - * underlying hardware supports. - * - * Methods available for AMD GPU driver for resetting the device. Not all - * methods are suitable for every device. User can override the method using - * module parameter `reset_method`. - */ -enum amd_reset_method { - AMD_RESET_METHOD_NONE = -1, - AMD_RESET_METHOD_LEGACY = 0, - AMD_RESET_METHOD_MODE0, - AMD_RESET_METHOD_MODE1, - AMD_RESET_METHOD_MODE2, - AMD_RESET_METHOD_LINK, - AMD_RESET_METHOD_BACO, - AMD_RESET_METHOD_PCI, - AMD_RESET_METHOD_ON_INIT, -}; - struct amdgpu_video_codec_info { u32 codec_type; u32 max_width; @@ -1111,6 +1075,7 @@ struct amdgpu_device { long psp_timeout; uint64_t unique_id; + uint8_t unitid; uint64_t df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS]; /* enable runtime pm on the device */ @@ -1372,6 +1337,8 @@ int emu_soc_asic_init(struct amdgpu_device *adev); #define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8)) #define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16)) +#include "amdgpu_reset.h" + /* * ASICs macro. */ @@ -1461,6 +1428,8 @@ ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring); ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset); void amdgpu_sdma_set_vm_pte_scheds(struct amdgpu_device *adev, const struct amdgpu_vm_pte_funcs *vm_pte_funcs); +void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev, + const struct amdgpu_buffer_funcs *buffer_funcs); /* atpx handler */ #if defined(CONFIG_VGA_SWITCHEROO) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index 62807b65f2af..9014678d75ab 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -228,7 +228,6 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) u64 acp_base; u32 val = 0; u32 count = 0; - struct i2s_platform_data *i2s_pdata = NULL; struct amdgpu_device *adev = ip_block->adev; @@ -272,18 +271,18 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) goto failure; } - i2s_pdata = kzalloc_objs(struct i2s_platform_data, 1); - if (!i2s_pdata) { + adev->acp.i2s_pdata = kzalloc_objs(struct i2s_platform_data, 1); + if (!adev->acp.i2s_pdata) { r = -ENOMEM; goto failure; } - i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | - DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; - i2s_pdata[0].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; - i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; - i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; - i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; + adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + adev->acp.i2s_pdata[0].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; + adev->acp.i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; + adev->acp.i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; + adev->acp.i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; adev->acp.acp_res[0].name = "acp2x_dma"; adev->acp.acp_res[0].flags = IORESOURCE_MEM; @@ -311,7 +310,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) adev->acp.acp_cell[1].id = 1; adev->acp.acp_cell[1].num_resources = 1; adev->acp.acp_cell[1].resources = &adev->acp.acp_res[1]; - adev->acp.acp_cell[1].platform_data = &i2s_pdata[0]; + adev->acp.acp_cell[1].platform_data = &adev->acp.i2s_pdata[0]; adev->acp.acp_cell[1].pdata_size = sizeof(struct i2s_platform_data); r = mfd_add_devices(adev->acp.parent, 0, adev->acp.acp_cell, 2, NULL, 0, NULL); if (r) @@ -336,53 +335,53 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) goto failure; } - i2s_pdata = kzalloc_objs(struct i2s_platform_data, 3); - if (!i2s_pdata) { + adev->acp.i2s_pdata = kzalloc_objs(struct i2s_platform_data, 3); + if (!adev->acp.i2s_pdata) { r = -ENOMEM; goto failure; } switch (adev->asic_type) { case CHIP_STONEY: - i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | - DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; break; default: - i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; + adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; } - i2s_pdata[0].cap = DWC_I2S_PLAY; - i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; - i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_PLAY_REG_OFFSET; - i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_PLAY_REG_OFFSET; + adev->acp.i2s_pdata[0].cap = DWC_I2S_PLAY; + adev->acp.i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; + adev->acp.i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_PLAY_REG_OFFSET; + adev->acp.i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_PLAY_REG_OFFSET; switch (adev->asic_type) { case CHIP_STONEY: - i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | - DW_I2S_QUIRK_COMP_PARAM1 | - DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + adev->acp.i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_COMP_PARAM1 | + DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; break; default: - i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | - DW_I2S_QUIRK_COMP_PARAM1; + adev->acp.i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_COMP_PARAM1; } - i2s_pdata[1].cap = DWC_I2S_RECORD; - i2s_pdata[1].snd_rates = SNDRV_PCM_RATE_8000_96000; - i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; - i2s_pdata[1].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; + adev->acp.i2s_pdata[1].cap = DWC_I2S_RECORD; + adev->acp.i2s_pdata[1].snd_rates = SNDRV_PCM_RATE_8000_96000; + adev->acp.i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; + adev->acp.i2s_pdata[1].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; - i2s_pdata[2].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; + adev->acp.i2s_pdata[2].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; switch (adev->asic_type) { case CHIP_STONEY: - i2s_pdata[2].quirks |= DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + adev->acp.i2s_pdata[2].quirks |= DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; break; default: break; } - i2s_pdata[2].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; - i2s_pdata[2].snd_rates = SNDRV_PCM_RATE_8000_96000; - i2s_pdata[2].i2s_reg_comp1 = ACP_BT_COMP1_REG_OFFSET; - i2s_pdata[2].i2s_reg_comp2 = ACP_BT_COMP2_REG_OFFSET; + adev->acp.i2s_pdata[2].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; + adev->acp.i2s_pdata[2].snd_rates = SNDRV_PCM_RATE_8000_96000; + adev->acp.i2s_pdata[2].i2s_reg_comp1 = ACP_BT_COMP1_REG_OFFSET; + adev->acp.i2s_pdata[2].i2s_reg_comp2 = ACP_BT_COMP2_REG_OFFSET; adev->acp.acp_res[0].name = "acp2x_dma"; adev->acp.acp_res[0].flags = IORESOURCE_MEM; @@ -420,21 +419,21 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) adev->acp.acp_cell[1].id = 1; adev->acp.acp_cell[1].num_resources = 1; adev->acp.acp_cell[1].resources = &adev->acp.acp_res[1]; - adev->acp.acp_cell[1].platform_data = &i2s_pdata[0]; + adev->acp.acp_cell[1].platform_data = &adev->acp.i2s_pdata[0]; adev->acp.acp_cell[1].pdata_size = sizeof(struct i2s_platform_data); adev->acp.acp_cell[2].name = "designware-i2s"; adev->acp.acp_cell[2].id = 2; adev->acp.acp_cell[2].num_resources = 1; adev->acp.acp_cell[2].resources = &adev->acp.acp_res[2]; - adev->acp.acp_cell[2].platform_data = &i2s_pdata[1]; + adev->acp.acp_cell[2].platform_data = &adev->acp.i2s_pdata[1]; adev->acp.acp_cell[2].pdata_size = sizeof(struct i2s_platform_data); adev->acp.acp_cell[3].name = "designware-i2s"; adev->acp.acp_cell[3].id = 3; adev->acp.acp_cell[3].num_resources = 1; adev->acp.acp_cell[3].resources = &adev->acp.acp_res[3]; - adev->acp.acp_cell[3].platform_data = &i2s_pdata[2]; + adev->acp.acp_cell[3].platform_data = &adev->acp.i2s_pdata[2]; adev->acp.acp_cell[3].pdata_size = sizeof(struct i2s_platform_data); r = mfd_add_devices(adev->acp.parent, 0, adev->acp.acp_cell, ACP_DEVS, NULL, 0, NULL); @@ -491,7 +490,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) return 0; failure: - kfree(i2s_pdata); + kfree(adev->acp.i2s_pdata); kfree(adev->acp.acp_res); kfree(adev->acp.acp_cell); kfree(adev->acp.acp_genpd); @@ -509,6 +508,7 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) u32 val = 0; u32 count = 0; struct amdgpu_device *adev = ip_block->adev; + int ret = 0; /* return early if no ACP */ if (!adev->acp.acp_genpd) { @@ -530,7 +530,8 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) break; if (--count == 0) { dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto out; } udelay(100); } @@ -547,20 +548,24 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) break; if (--count == 0) { dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto out; } udelay(100); } - +out: device_for_each_child(adev->acp.parent, NULL, acp_genpd_remove_device); mfd_remove_devices(adev->acp.parent); + kfree(adev->acp.i2s_pdata); kfree(adev->acp.acp_res); + pm_genpd_remove(&adev->acp.acp_genpd->gpd); kfree(adev->acp.acp_genpd); + adev->acp.acp_genpd = NULL; kfree(adev->acp.acp_cell); - return 0; + return ret; } static int acp_suspend(struct amdgpu_ip_block *ip_block) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h index a288ce25c176..13b48c582314 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h @@ -35,6 +35,7 @@ struct amdgpu_acp { struct mfd_cell *acp_cell; struct resource *acp_res; struct acp_pm_domain *acp_genpd; + struct i2s_platform_data *i2s_pdata; }; extern const struct amdgpu_ip_block_version acp_ip_block; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 9783a3cefb04..da325863ad76 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -558,7 +558,7 @@ uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev) int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, struct amdgpu_device **dmabuf_adev, - uint64_t *bo_size, void *metadata_buffer, + uint64_t *bo_size, void **metadata_buffer, size_t buffer_size, uint32_t *metadata_size, uint32_t *flags, int8_t *xcp_id) { @@ -593,9 +593,24 @@ int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, *dmabuf_adev = adev; if (bo_size) *bo_size = amdgpu_bo_size(bo); - if (metadata_buffer) - r = amdgpu_bo_get_metadata(bo, metadata_buffer, buffer_size, - metadata_size, &metadata_flags); + if (metadata_buffer) { + /* first get metadata_size by buffer = NULL */ + r = amdgpu_bo_get_metadata(bo, NULL, 0, + metadata_size, NULL); + + /* user buf_size is bigger than bo metadata_size + * allocate a buf at kernel space and copy */ + if (*metadata_size <= buffer_size) { + *metadata_buffer = kzalloc(*metadata_size, GFP_KERNEL); + + if (!*metadata_buffer) + return -ENOMEM; + + r = amdgpu_bo_get_metadata(bo, *metadata_buffer, *metadata_size, + NULL, &metadata_flags); + } else + r = -EINVAL; + } if (flags) { *flags = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? KFD_IOC_ALLOC_MEM_FLAGS_VRAM diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 2b4108f83f48..e443a7277299 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -38,6 +38,8 @@ #include "amdgpu_vm.h" #include "amdgpu_xcp.h" #include "kfd_topology.h" +#include "amdgpu_ptl.h" + extern uint64_t amdgpu_amdkfd_total_mem_size; enum TLB_FLUSH_TYPE { @@ -260,7 +262,7 @@ uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct amdgpu_device *adev); uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev); int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, struct amdgpu_device **dmabuf_adev, - uint64_t *bo_size, void *metadata_buffer, + uint64_t *bo_size, void **metadata_buffer, size_t buffer_size, uint32_t *metadata_size, uint32_t *flags, int8_t *xcp_id); int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min); @@ -435,8 +437,10 @@ int kgd2kfd_check_and_lock_kfd(struct kfd_dev *kfd); void kgd2kfd_unlock_kfd(struct kfd_dev *kfd); int kgd2kfd_start_sched(struct kfd_dev *kfd, uint32_t node_id); int kgd2kfd_start_sched_all_nodes(struct kfd_dev *kfd); +int amdgpu_amdkfd_start_sched_all(struct amdgpu_device *adev); int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id); int kgd2kfd_stop_sched_all_nodes(struct kfd_dev *kfd); +int amdgpu_amdkfd_stop_sched_all(struct amdgpu_device *adev); bool kgd2kfd_compute_active(struct kfd_dev *kfd, uint32_t node_id); bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry, bool retry_fault); @@ -533,6 +537,11 @@ static inline int kgd2kfd_start_sched_all_nodes(struct kfd_dev *kfd) return 0; } +static inline int amdgpu_amdkfd_start_sched_all(struct amdgpu_device *adev) +{ + return 0; +} + static inline int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id) { return 0; @@ -543,6 +552,11 @@ static inline int kgd2kfd_stop_sched_all_nodes(struct kfd_dev *kfd) return 0; } +static inline int amdgpu_amdkfd_stop_sched_all(struct amdgpu_device *adev) +{ + return 0; +} + static inline bool kgd2kfd_compute_active(struct kfd_dev *kfd, uint32_t node_id) { return false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c index f35947be3763..6ed399163547 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c @@ -520,6 +520,16 @@ static uint32_t kgd_gfx_v9_4_3_hqd_sdma_get_doorbell(struct amdgpu_device *adev, return is_active ? doorbell_off >> 2 : 0; } +static uint32_t kgd_v9_4_3_ptl_ctrl(struct amdgpu_device *adev, + uint32_t cmd, + uint32_t *ptl_state, + enum amdgpu_ptl_fmt *fmt1, + enum amdgpu_ptl_fmt *fmt2) +{ + return amdgpu_ptl_perf_monitor_ctrl(adev, cmd, + ptl_state, fmt1, fmt2); +} + const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = { .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, .set_pasid_vmid_mapping = kgd_gfx_v9_4_3_set_pasid_vmid_mapping, @@ -555,5 +565,6 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = { .clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch, .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr, .hqd_reset = kgd_gfx_v9_hqd_reset, - .hqd_sdma_get_doorbell = kgd_gfx_v9_4_3_hqd_sdma_get_doorbell + .hqd_sdma_get_doorbell = kgd_gfx_v9_4_3_hqd_sdma_get_doorbell, + .ptl_ctrl = kgd_v9_4_3_ptl_ctrl }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 72a5a29e63f6..35fe2c974699 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1914,13 +1914,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( mutex_lock(&mem->lock); - /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */ - if (mem->alloc_flags & - (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | - KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { - amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); - } - mapped_to_gpu_memory = mem->mapped_to_gpu_memory; is_imported = mem->is_imported; mutex_unlock(&mem->lock); @@ -1934,6 +1927,15 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( return -EBUSY; } + /* At this point the BO is guaranteed to be freed, so unpin the + * MMIO/DOORBELL BOs that were pinned during allocation. + */ + if (mem->alloc_flags & + (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | + KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { + amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); + } + /* Make sure restore workers don't access the BO any more */ mutex_lock(&process_info->lock); if (!list_empty(&mem->validate_list)) @@ -2999,7 +3001,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu * /* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO * validations above would invalidate DMABuf imports again. */ - ret = process_validate_vms(process_info, &exec.ticket); + ret = process_validate_vms(process_info, drm_exec_ticket(&exec)); if (ret) { pr_debug("Validating VMs failed, ret: %d\n", ret); goto validate_map_fail; @@ -3040,7 +3042,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu * goto validate_map_fail; } - ret = amdgpu_vm_handle_moved(adev, peer_vm, &exec.ticket); + ret = amdgpu_vm_handle_moved(adev, peer_vm, drm_exec_ticket(&exec)); if (ret) { dev_dbg(adev->dev, "Memory eviction: handle moved failed, pid %8d. Try again.\n", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c index 3698dd0330ff..acd22bff1882 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c @@ -36,6 +36,21 @@ #include "atombios_encoders.h" #include "bif/bif_4_1_d.h" +/* VBIOS-reported table size is unchecked against the image; cap iterations and + * adev->i2c_bus[] indexing to AMDGPU_MAX_I2C_BUS. + */ +static int amdgpu_atombios_gpio_i2c_num_entries(uint16_t size) +{ + u32 bytes; + + if (size < sizeof(ATOM_COMMON_TABLE_HEADER)) + return 0; + + bytes = size - sizeof(ATOM_COMMON_TABLE_HEADER); + return (int)min_t(u32, bytes / sizeof(ATOM_GPIO_I2C_ASSIGMENT), + AMDGPU_MAX_I2C_BUS); +} + static struct amdgpu_i2c_bus_rec amdgpu_atombios_get_bus_rec_for_i2c_gpio(ATOM_GPIO_I2C_ASSIGMENT *gpio) { struct amdgpu_i2c_bus_rec i2c; @@ -96,8 +111,7 @@ struct amdgpu_i2c_bus_rec amdgpu_atombios_lookup_i2c_gpio(struct amdgpu_device * if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { @@ -127,8 +141,7 @@ void amdgpu_atombios_i2c_init(struct amdgpu_device *adev) if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { @@ -158,8 +171,7 @@ void amdgpu_atombios_oem_i2c_init(struct amdgpu_device *adev, u8 i2c_id) if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index 35d04e69aec0..aa039e148a5e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -33,6 +33,7 @@ #include <linux/pci.h> #include <linux/slab.h> #include <linux/acpi.h> +#include <linux/vgaarb.h> /* * BIOS. */ @@ -467,7 +468,8 @@ static bool amdgpu_prefer_rom_resource(struct amdgpu_device *adev) { struct resource *res = &adev->pdev->resource[PCI_ROM_RESOURCE]; - return (res->flags & IORESOURCE_ROM_SHADOW); + return (res->flags & IORESOURCE_ROM_SHADOW) || + adev->pdev == vga_default_device(); } static bool amdgpu_get_bios_dgpu(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c index 4e6e390854e6..d5e59c24d907 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c @@ -153,7 +153,7 @@ int amdgpu_cper_entry_fill_fatal_section(struct amdgpu_device *adev, FATAL_SEC_OFFSET(hdr->sec_cnt, idx)); amdgpu_cper_entry_fill_section_desc(adev, section_desc, false, false, - CPER_SEV_FATAL, CRASHDUMP, FATAL_SEC_LEN, + CPER_SEV_FATAL_UNCORRECTED, CRASHDUMP, FATAL_SEC_LEN, FATAL_SEC_OFFSET(hdr->sec_cnt, idx)); section->body.reg_ctx_type = CPER_CTX_TYPE_CRASH; @@ -215,7 +215,7 @@ int amdgpu_cper_entry_fill_bad_page_threshold_section(struct amdgpu_device *adev NONSTD_SEC_OFFSET(hdr->sec_cnt, idx)); amdgpu_cper_entry_fill_section_desc(adev, section_desc, true, false, - CPER_SEV_FATAL, RUNTIME, NONSTD_SEC_LEN, + CPER_SEV_FATAL_UNCORRECTED, RUNTIME, NONSTD_SEC_LEN, NONSTD_SEC_OFFSET(hdr->sec_cnt, idx)); section->hdr.valid_bits.err_info_cnt = 1; @@ -312,7 +312,7 @@ int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev, reg_data.synd_lo = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]); reg_data.synd_hi = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]); - amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL); + amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL_UNCORRECTED); ret = amdgpu_cper_entry_fill_fatal_section(adev, fatal, 0, reg_data); if (ret) return ret; @@ -337,7 +337,7 @@ int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev) amdgpu_cper_entry_fill_hdr(adev, bp_threshold, AMDGPU_CPER_TYPE_BP_THRESHOLD, - CPER_SEV_FATAL); + CPER_SEV_FATAL_UNCORRECTED); ret = amdgpu_cper_entry_fill_bad_page_threshold_section(adev, bp_threshold, 0); if (ret) return ret; @@ -353,14 +353,14 @@ static enum cper_error_severity amdgpu_aca_err_type_to_cper_sev(struct amdgpu_de { switch (aca_err_type) { case ACA_ERROR_TYPE_UE: - return CPER_SEV_FATAL; + return CPER_SEV_FATAL_UNCORRECTED; case ACA_ERROR_TYPE_CE: return CPER_SEV_NON_FATAL_CORRECTED; case ACA_ERROR_TYPE_DEFERRED: return CPER_SEV_NON_FATAL_UNCORRECTED; default: dev_err(adev->dev, "Unknown ACA error type!\n"); - return CPER_SEV_FATAL; + return CPER_SEV_FATAL_UNCORRECTED; } } @@ -484,7 +484,7 @@ calc: void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) { - u64 pos, wptr_old, rptr; + u64 pos, wptr_old, rptr, next_rptr; int rec_cnt_dw = count >> 2; u32 chunk, ent_sz; u8 *s = (u8 *)src; @@ -525,9 +525,19 @@ void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) do { ent_sz = amdgpu_cper_ring_get_ent_sz(ring, pos); - - rptr += (ent_sz >> 2); - rptr &= ring->ptr_mask; + next_rptr = rptr; + if (ent_sz >= sizeof(u32)) + next_rptr = (rptr + (ent_sz >> 2)) & ring->ptr_mask; + + if (next_rptr == rptr) { + /* Corrupt entry size, reset the ring to avoid an infinite loop. */ + rptr = ring->wptr; + *ring->rptr_cpu_addr = rptr; + ring->count_dw = (ring->ring_size - 4) >> 2; + goto out_unlock; + } + + rptr = next_rptr; *ring->rptr_cpu_addr = rptr; pos = rptr; @@ -536,6 +546,8 @@ void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) if (ring->count_dw >= rec_cnt_dw) ring->count_dw -= rec_cnt_dw; + +out_unlock: mutex_unlock(&ring->adev->cper.ring_lock); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index b24d5d21be5f..e714cee2997a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -60,11 +60,6 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, if (!p->ctx) return -EINVAL; - if (atomic_read(&p->ctx->guilty)) { - amdgpu_ctx_put(p->ctx); - return -ECANCELED; - } - amdgpu_sync_create(&p->sync); drm_exec_init(&p->exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0); @@ -252,13 +247,17 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, goto free_partial_kdata; break; + case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: + if (size < sizeof(struct drm_amdgpu_cs_chunk_cp_gfx_shadow)) + goto free_partial_kdata; + break; + case AMDGPU_CHUNK_ID_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_IN: case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: - case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: break; default: @@ -850,7 +849,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, struct amdgpu_vm *vm = &fpriv->vm; struct amdgpu_bo_list_entry *e; struct drm_gem_object *obj; - unsigned long index; unsigned int i; int r; @@ -961,7 +959,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, goto out_free_user_pages; } - drm_exec_for_each_locked_object(&p->exec, index, obj) { + drm_exec_for_each_locked_object(&p->exec, obj) { r = amdgpu_cs_bo_validate(p, gem_to_amdgpu_bo(obj)); if (unlikely(r)) goto out_free_user_pages; @@ -1100,7 +1098,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) if (p->gang_size > 1 && !adev->vm_manager.concurrent_flush) { for (i = 0; i < p->gang_size; ++i) { struct drm_sched_entity *entity = p->entities[i]; - struct drm_gpu_scheduler *sched = entity->rq->sched; + struct drm_gpu_scheduler *sched = + container_of(entity->rq, typeof(*sched), rq); struct amdgpu_ring *ring = to_amdgpu_ring(sched); if (amdgpu_vmid_uses_reserved(vm, ring->vm_hub)) @@ -1157,7 +1156,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) return r; } - r = amdgpu_vm_handle_moved(adev, vm, &p->exec.ticket); + r = amdgpu_vm_handle_moved(adev, vm, drm_exec_ticket(&p->exec)); if (r) return r; @@ -1200,7 +1199,6 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) struct drm_gpu_scheduler *sched; struct drm_gem_object *obj; struct dma_fence *fence; - unsigned long index; unsigned int i; int r; @@ -1211,7 +1209,7 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) return r; } - drm_exec_for_each_locked_object(&p->exec, index, obj) { + drm_exec_for_each_locked_object(&p->exec, obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); struct dma_resv *resv = bo->tbo.base.resv; @@ -1231,7 +1229,8 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) return r; } - sched = p->gang_leader->base.entity->rq->sched; + sched = container_of(p->gang_leader->base.entity->rq, typeof(*sched), + rq); while ((fence = amdgpu_sync_get_fence(&p->sync))) { struct drm_sched_fence *s_fence = to_drm_sched_fence(fence); @@ -1277,9 +1276,9 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; struct amdgpu_job *leader = p->gang_leader; + struct amdgpu_vm *vm = &fpriv->vm; struct amdgpu_bo_list_entry *e; struct drm_gem_object *gobj; - unsigned long index; unsigned int i; uint64_t seq; int r; @@ -1322,14 +1321,15 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, amdgpu_hmm_range_free(e->range); e->range = NULL; } - if (r) { + + if (r || !list_empty(&vm->individual.needs_update)) { r = -EAGAIN; mutex_unlock(&p->adev->notifier_lock); return r; } p->fence = dma_fence_get(&leader->base.s_fence->finished); - drm_exec_for_each_locked_object(&p->exec, index, gobj) { + drm_exec_for_each_locked_object(&p->exec, gobj) { ttm_bo_move_to_lru_tail_unlocked(&gem_to_amdgpu_bo(gobj)->tbo); @@ -1360,7 +1360,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, cs->out.handle = seq; leader->uf_sequence = seq; - amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->exec.ticket); + amdgpu_vm_bo_trace_cs(&fpriv->vm, drm_exec_ticket(&p->exec)); for (i = 0; i < p->gang_size; ++i) { amdgpu_job_free_resources(p->jobs[i]); trace_amdgpu_cs_ioctl(p->jobs[i]); @@ -1795,7 +1795,7 @@ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, *map = mapping; /* Double check that the BO is reserved by this CS */ - if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->exec.ticket) + if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != drm_exec_ticket(&parser->exec)) return -EINVAL; /* Make sure VRAM is allocated contigiously */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 7af86a32c0c5..ce35b415093d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -255,7 +255,7 @@ static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip, } r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds, - &ctx->guilty); + NULL); if (r) goto error_free_entity; @@ -326,7 +326,6 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, struct drm_file *filp, struct amdgpu_ctx *ctx) { struct amdgpu_fpriv *fpriv = filp->driver_priv; - u32 current_stable_pstate; int r; r = amdgpu_ctx_priority_permit(filp, priority); @@ -344,36 +343,21 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, ctx->generation = amdgpu_vm_generation(mgr->adev, &fpriv->vm); ctx->init_priority = priority; ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET; - - r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); - if (r) - return r; - - if (mgr->adev->pm.stable_pstate_ctx) - ctx->stable_pstate = mgr->adev->pm.stable_pstate_ctx->stable_pstate; - else - ctx->stable_pstate = current_stable_pstate; + ctx->stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE; return 0; } -static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, - u32 stable_pstate) +static int __amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, + u32 stable_pstate) { struct amdgpu_device *adev = ctx->mgr->adev; enum amd_dpm_forced_level level; + struct amdgpu_ctx *current_ctx; u32 current_stable_pstate; - int r; + int r = 0; - mutex_lock(&adev->pm.stable_pstate_ctx_lock); - if (adev->pm.stable_pstate_ctx && adev->pm.stable_pstate_ctx != ctx) { - r = -EBUSY; - goto done; - } - - r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); - if (r || (stable_pstate == current_stable_pstate)) - goto done; + lockdep_assert_held(&adev->pm.stable_pstate_ctx_lock); switch (stable_pstate) { case AMDGPU_CTX_STABLE_PSTATE_NONE: @@ -392,17 +376,41 @@ static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK; break; default: - r = -EINVAL; - goto done; + return -EINVAL; } + current_ctx = adev->pm.stable_pstate_ctx; + if (current_ctx && current_ctx != ctx) + return -EBUSY; + + r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); + if (r || current_stable_pstate == stable_pstate) + return r; + r = amdgpu_dpm_force_performance_level(adev, level); + if (r) + return r; - if (level == AMD_DPM_FORCED_LEVEL_AUTO) - adev->pm.stable_pstate_ctx = NULL; - else + if (!current_ctx) { adev->pm.stable_pstate_ctx = ctx; -done: + /* + * Serialized by context taking ownership for the first time + * while holding adev->pm.stable_pstate_ctx_lock). + */ + WRITE_ONCE(ctx->stable_pstate, current_stable_pstate); + } + + return 0; +} + +static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, + u32 stable_pstate) +{ + struct amdgpu_device *adev = ctx->mgr->adev; + int r; + + mutex_lock(&adev->pm.stable_pstate_ctx_lock); + r = __amdgpu_ctx_set_stable_pstate(ctx, stable_pstate); mutex_unlock(&adev->pm.stable_pstate_ctx_lock); return r; @@ -428,7 +436,12 @@ static void amdgpu_ctx_fini(struct kref *ref) } if (drm_dev_enter(adev_to_drm(adev), &idx)) { - amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate); + mutex_lock(&adev->pm.stable_pstate_ctx_lock); + if (adev->pm.stable_pstate_ctx == ctx) { + __amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate); + adev->pm.stable_pstate_ctx = NULL; + } + mutex_unlock(&adev->pm.stable_pstate_ctx_lock); drm_dev_exit(idx); } @@ -579,6 +592,27 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev, #define AMDGPU_RAS_COUNTE_DELAY_MS 3000 +static bool amdgpu_ctx_guilty(struct amdgpu_ctx *ctx) +{ + int i, j, r; + + for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { + for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { + struct amdgpu_ctx_entity *ctx_entity; + + ctx_entity = ctx->entities[i][j]; + if (!ctx_entity) + continue; + + r = drm_sched_entity_error(&ctx_entity->entity); + if (r == -ETIME) + return true; + } + } + + return false; +} + static int amdgpu_ctx_query2(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, uint32_t id, union drm_amdgpu_ctx_out *out) @@ -607,7 +641,7 @@ static int amdgpu_ctx_query2(struct amdgpu_device *adev, if (ctx->generation != amdgpu_vm_generation(adev, &fpriv->vm)) out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST; - if (atomic_read(&ctx->guilty)) + if (amdgpu_ctx_guilty(ctx)) out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY; if (amdgpu_in_reset(adev)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h index cf8d700a22fe..e444b2088d40 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h @@ -50,7 +50,6 @@ struct amdgpu_ctx { int32_t init_priority; int32_t override_priority; uint32_t stable_pstate; - atomic_t guilty; bool preamble_presented; uint64_t generation; unsigned long ras_counter_ce; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index 092fd3309099..389bad724273 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -2049,7 +2049,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) /* swap out the old fences */ amdgpu_ib_preempt_fences_swap(ring, fences); - amdgpu_fence_driver_force_completion(ring); + amdgpu_fence_driver_force_completion(ring, NULL); /* resubmit unfinished jobs */ amdgpu_ib_preempt_job_recovery(&ring->sched); @@ -2126,6 +2126,9 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) debugfs_create_x32("amdgpu_smu_debug", 0600, root, &adev->pm.smu_debug_mask); + debugfs_create_x64("unique_id", 0444, root, &adev->unique_id); + debugfs_create_x8("unitid", 0444, root, &adev->unitid); + ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev, &fops_ib_preempt); if (IS_ERR(ent)) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c index d386bc775d03..e77db76b48b8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c @@ -22,8 +22,9 @@ * */ -#include <generated/utsrelease.h> #include <linux/devcoredump.h> +#include <linux/utsname.h> +#include <drm/drm_exec.h> #include "amdgpu_dev_coredump.h" #include "atom.h" @@ -207,28 +208,143 @@ static void amdgpu_devcoredump_fw_info(struct amdgpu_device *adev, } } +static void +amdgpu_devcoredump_print_ibs(struct drm_printer *p, + struct amdgpu_coredump_info *coredump, + bool sizing_pass) +{ + struct amdgpu_device *adev = coredump->adev; + struct amdgpu_bo_va_mapping *mapping; + struct amdgpu_bo *abo; + struct drm_exec exec; + struct amdgpu_vm *vm; + u32 *ib_content; + u64 va_start, offset; + u8 *kptr; + u32 off; + int r; + + /* + * On the sizing pass there is no VM to look up and no BO to lock; the + * size estimate doesn't depend on whether the IB BOs are reachable. + * Just emit the per-IB headers (the content is not written anywhere). + */ + if (sizing_pass) { + for (int i = 0; i < coredump->num_ibs; i++) { + drm_printf(p, "\nIB #%d 0x%llx %d dw\n", i, + coredump->ibs[i].gpu_addr, + coredump->ibs[i].ib_size_dw); + } + return; + } + + /* + * Lock the VM root PD and every IB BO together in a single drm_exec + * ticket. Reserving the IB BOs one by one while the root PD is held + * would be a recursive reservation_ww_class_mutex acquire without a + * ww_acquire_ctx, which trips lockdep and self-deadlocks for IB BOs + * that share their dma_resv with the root PD (always-valid BOs). + */ + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 1 + coredump->num_ibs); + drm_exec_until_all_locked(&exec) { + vm = amdgpu_vm_lock_by_pasid(adev, coredump->pasid, &exec); + if (!vm) + goto unlock; + + for (int i = 0; i < coredump->num_ibs; i++) { + u64 pfn = (coredump->ibs[i].gpu_addr & + AMDGPU_GMC_HOLE_MASK) / AMDGPU_GPU_PAGE_SIZE; + + mapping = amdgpu_vm_bo_lookup_mapping(vm, pfn); + if (!mapping) + continue; + + abo = mapping->bo_va->base.bo; + r = drm_exec_lock_obj(&exec, &abo->tbo.base); + drm_exec_retry_on_contention(&exec); + if (r) + goto unlock; + } + } + + for (int i = 0; i < coredump->num_ibs; i++) { + bool emit_content = false; + + ib_content = kvmalloc_array(coredump->ibs[i].ib_size_dw, 4, + GFP_KERNEL); + if (!ib_content) + continue; + + va_start = coredump->ibs[i].gpu_addr & AMDGPU_GMC_HOLE_MASK; + mapping = amdgpu_vm_bo_lookup_mapping(vm, + va_start / AMDGPU_GPU_PAGE_SIZE); + if (!mapping) + goto output_ib_content; + + abo = mapping->bo_va->base.bo; + offset = va_start - mapping->start * AMDGPU_GPU_PAGE_SIZE; + + if (abo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) { + struct amdgpu_res_cursor cursor; + + off = 0; + + if (abo->tbo.resource->mem_type != TTM_PL_VRAM) + goto output_ib_content; + + amdgpu_res_first(abo->tbo.resource, offset, + coredump->ibs[i].ib_size_dw * 4, &cursor); + while (cursor.remaining) { + amdgpu_device_mm_access(adev, cursor.start / 4, + &ib_content[off], cursor.size / 4, + false); + off += cursor.size; + amdgpu_res_next(&cursor, cursor.size); + } + emit_content = true; + } else { + r = ttm_bo_kmap(&abo->tbo, 0, PFN_UP(abo->tbo.base.size), + &abo->kmap); + if (r) + goto output_ib_content; + + kptr = amdgpu_bo_kptr(abo); + kptr += offset; + memcpy(ib_content, kptr, coredump->ibs[i].ib_size_dw * 4); + + amdgpu_bo_kunmap(abo); + emit_content = true; + } + +output_ib_content: + drm_printf(p, "\nIB #%d 0x%llx %d dw\n", i, + coredump->ibs[i].gpu_addr, coredump->ibs[i].ib_size_dw); + if (emit_content) { + for (int j = 0; j < coredump->ibs[i].ib_size_dw; j++) + drm_printf(p, "0x%08x\n", ib_content[j]); + } + kvfree(ib_content); + } + +unlock: + drm_exec_fini(&exec); +} + static ssize_t amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_info *coredump) { - struct amdgpu_device *adev = coredump->adev; struct drm_printer p; struct drm_print_iterator iter; struct amdgpu_vm_fault_info *fault_info; - struct amdgpu_bo_va_mapping *mapping; struct amdgpu_ip_block *ip_block; - struct amdgpu_res_cursor cursor; - struct amdgpu_bo *abo, *root; - uint64_t va_start, offset; struct amdgpu_ring *ring; - struct amdgpu_vm *vm; - u32 *ib_content; - uint8_t *kptr; - int ver, i, j, r; + int ver, i, j; u32 ring_idx, off; bool sizing_pass; sizing_pass = buffer == NULL; iter.data = buffer; + iter.start = 0; iter.offset = 0; iter.remain = count; @@ -236,7 +352,7 @@ amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_inf drm_printf(&p, "**** AMDGPU Device Coredump ****\n"); drm_printf(&p, "version: " AMDGPU_COREDUMP_VERSION "\n"); - drm_printf(&p, "kernel: " UTS_RELEASE "\n"); + drm_printf(&p, "kernel: %s\n", init_utsname()->release); drm_printf(&p, "module: " KBUILD_MODNAME "\n"); drm_printf(&p, "time: %ptSp\n", &coredump->reset_time); @@ -342,86 +458,8 @@ amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_inf else if (coredump->reset_vram_lost) drm_printf(&p, "VRAM is lost due to GPU reset!\n"); - if (coredump->num_ibs) { - /* Don't try to lookup the VM or map the BOs when calculating the - * size required to store the devcoredump. - */ - if (sizing_pass) - vm = NULL; - else - vm = amdgpu_vm_lock_by_pasid(adev, &root, coredump->pasid); - - for (int i = 0; i < coredump->num_ibs && (sizing_pass || vm); i++) { - ib_content = kvmalloc_array(coredump->ibs[i].ib_size_dw, 4, - GFP_KERNEL); - if (!ib_content) - continue; - - /* vm=NULL can only happen when 'sizing_pass' is true. Skip to the - * drm_printf() calls (ib_content doesn't need to be initialized - * as its content won't be written anywhere). - */ - if (!vm) - goto output_ib_content; - - va_start = coredump->ibs[i].gpu_addr & AMDGPU_GMC_HOLE_MASK; - mapping = amdgpu_vm_bo_lookup_mapping(vm, va_start / AMDGPU_GPU_PAGE_SIZE); - if (!mapping) - goto free_ib_content; - - offset = va_start - (mapping->start * AMDGPU_GPU_PAGE_SIZE); - abo = amdgpu_bo_ref(mapping->bo_va->base.bo); - r = amdgpu_bo_reserve(abo, false); - if (r) - goto free_ib_content; - - if (abo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) { - off = 0; - - if (abo->tbo.resource->mem_type != TTM_PL_VRAM) - goto unreserve_abo; - - amdgpu_res_first(abo->tbo.resource, offset, - coredump->ibs[i].ib_size_dw * 4, - &cursor); - while (cursor.remaining) { - amdgpu_device_mm_access(adev, cursor.start / 4, - &ib_content[off], cursor.size / 4, - false); - off += cursor.size; - amdgpu_res_next(&cursor, cursor.size); - } - } else { - r = ttm_bo_kmap(&abo->tbo, 0, - PFN_UP(abo->tbo.base.size), - &abo->kmap); - if (r) - goto unreserve_abo; - - kptr = amdgpu_bo_kptr(abo); - kptr += offset; - memcpy(ib_content, kptr, - coredump->ibs[i].ib_size_dw * 4); - - amdgpu_bo_kunmap(abo); - } - -output_ib_content: - drm_printf(&p, "\nIB #%d 0x%llx %d dw\n", - i, coredump->ibs[i].gpu_addr, coredump->ibs[i].ib_size_dw); - for (int j = 0; j < coredump->ibs[i].ib_size_dw; j++) - drm_printf(&p, "0x%08x\n", ib_content[j]); -unreserve_abo: - if (vm) - amdgpu_bo_unreserve(abo); -free_ib_content: - kvfree(ib_content); - } - if (vm) { - amdgpu_bo_unreserve(root); - amdgpu_bo_unref(&root); - } - } + if (coredump->num_ibs) + amdgpu_devcoredump_print_ibs(&p, coredump, sizing_pass); return count - iter.remain; } @@ -553,7 +591,7 @@ void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check, coredump->rings_dw = kzalloc(total_ring_size, GFP_NOWAIT); coredump->rings = kcalloc(ring_count, sizeof(struct amdgpu_coredump_ring), GFP_NOWAIT); if (coredump->rings && coredump->rings_dw) { - for (i = 0, off = 0, idx = 0; i < adev->num_rings; i++) { + for (i = 0, off = 0, idx = 0; i < adev->num_rings && idx < ring_count; i++) { ring = adev->rings[i]; if (atomic_read(&ring->fence_drv.last_seq) == ring->fence_drv.sync_seq && @@ -586,7 +624,7 @@ void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check, */ adev->coredump = coredump; /* Kick off coredump formatting to a worker thread. */ - queue_work(system_unbound_wq, &adev->coredump_work); + queue_work(system_dfl_wq, &adev->coredump_work); drm_info(dev, "AMDGPU device coredump file has been created\n"); drm_info(dev, "Check your /sys/class/drm/card%d/device/devcoredump/data\n", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index feab90e3efd1..53335ca96b1d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -717,7 +717,12 @@ void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos, if (!drm_dev_enter(adev_to_drm(adev), &idx)) return; - BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)); + if (!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)) { + dev_err(adev->dev, "unaligned pos/size (pos=0x%llx, size=0x%zx)\n", + pos, size); + drm_dev_exit(idx); + return; + } spin_lock_irqsave(&adev->mmio_idx_lock, flags); for (last = pos + size; pos < last; pos += 4) { @@ -1328,7 +1333,8 @@ static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) * It's unclear if this is a platform-specific or GPU-specific issue. * Disable ASPM on SI for the time being. */ - if (adev->family == AMDGPU_FAMILY_SI) + if (adev->family == AMDGPU_FAMILY_SI || + (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK) && adev->family == AMDGPU_FAMILY_VI)) return true; #if IS_ENABLED(CONFIG_X86) @@ -1913,6 +1919,20 @@ static void amdgpu_uid_fini(struct amdgpu_device *adev) adev->uid_info = NULL; } +static struct pci_dev *amdgpu_device_find_parent(struct amdgpu_device *adev) +{ + struct pci_dev *parent = adev->pdev; + + /* skip upstream/downstream switches internal to dGPU */ + while ((parent = pci_upstream_bridge(parent))) { + if (parent->vendor == PCI_VENDOR_ID_ATI) + continue; + break; + } + + return parent; +} + /** * amdgpu_device_ip_early_init - run early init for hardware IPs * @@ -2014,7 +2034,7 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) adev->flags |= AMD_IS_PX; if (!(adev->flags & AMD_IS_APU)) { - parent = pcie_find_root_port(adev->pdev); + parent = amdgpu_device_find_parent(adev); adev->has_pr3 = parent ? pci_pr3_present(parent) : false; } @@ -2221,7 +2241,6 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) { struct drm_sched_init_args args = { .ops = &amdgpu_sched_ops, - .num_rqs = DRM_SCHED_PRIORITY_COUNT, .timeout_wq = adev->reset_domain->wq, .dev = adev->dev, }; @@ -2460,7 +2479,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) if (r) goto init_failed; - amdgpu_ttm_set_buffer_funcs_status(adev, true); + amdgpu_ttm_enable_buffer_funcs(adev); /* Don't init kfd if whole hive need to be reset during init */ if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { @@ -3025,7 +3044,7 @@ static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry); for (i = adev->num_ip_blocks - 1; i >= 0; i--) { - if (!adev->ip_blocks[i].status.valid) + if (!adev->ip_blocks[i].status.valid || !adev->ip_blocks[i].status.hw) continue; /* displays are handled in phase1 */ if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) @@ -3148,7 +3167,7 @@ static int amdgpu_device_ip_suspend(struct amdgpu_device *adev) amdgpu_virt_request_full_gpu(adev, false); } - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_ttm_disable_buffer_funcs(adev); r = amdgpu_device_ip_suspend_phase1(adev); if (r) @@ -3363,7 +3382,7 @@ static int amdgpu_device_ip_resume(struct amdgpu_device *adev) r = amdgpu_device_ip_resume_phase2(adev); - amdgpu_ttm_set_buffer_funcs_status(adev, true); + amdgpu_ttm_enable_buffer_funcs(adev); if (r) return r; @@ -3648,6 +3667,7 @@ static int amdgpu_device_sys_interface_init(struct amdgpu_device *adev) amdgpu_reg_state_sysfs_init(adev); amdgpu_xcp_sysfs_init(adev); amdgpu_uma_sysfs_init(adev); + amdgpu_ptl_sysfs_init(adev); return r; } @@ -3664,6 +3684,7 @@ static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev) amdgpu_reg_state_sysfs_fini(adev); amdgpu_xcp_sysfs_fini(adev); amdgpu_uma_sysfs_fini(adev); + amdgpu_ptl_sysfs_fini(adev); } /** @@ -3701,7 +3722,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, adev->num_rings = 0; RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); adev->mman.buffer_funcs = NULL; - adev->mman.buffer_funcs_ring = NULL; + adev->mman.num_buffer_funcs_scheds = 0; adev->vm_manager.vm_pte_funcs = NULL; adev->vm_manager.vm_pte_num_scheds = 0; adev->gmc.gmc_funcs = NULL; @@ -3732,10 +3753,14 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->virt.vf_errors.lock); hash_init(adev->mn_hash); mutex_init(&adev->psp.mutex); + mutex_init(&adev->psp.ptl.mutex); mutex_init(&adev->notifier_lock); mutex_init(&adev->pm.stable_pstate_ctx_lock); mutex_init(&adev->benchmark_mutex); mutex_init(&adev->gfx.reset_sem_mutex); + + /* Associate locks with lockdep classes for ordering validation */ + amdgpu_lockdep_set_class(adev); /* Initialize the mutex for cleaner shader isolation between GFX and compute processes */ mutex_init(&adev->enforce_isolation_mutex); for (i = 0; i < MAX_XCP; ++i) { @@ -3747,6 +3772,8 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->gfx.workload_profile_mutex); mutex_init(&adev->vcn.workload_profile_mutex); + spin_lock_init(&adev->irq.lock); + amdgpu_device_init_apu_flags(adev); r = amdgpu_device_check_arguments(adev); @@ -3857,6 +3884,9 @@ int amdgpu_device_init(struct amdgpu_device *adev, * completed before the need for a different level is detected. */ amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT); + + amdgpu_device_check_iommu_direct_map(adev); + /* early init functions */ r = amdgpu_device_ip_early_init(adev); if (r) @@ -4114,8 +4144,6 @@ fence_driver_init: if (px) vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); - amdgpu_device_check_iommu_direct_map(adev); - adev->pm_nb.notifier_call = amdgpu_device_pm_notifier; r = register_pm_notifier(&adev->pm_nb); if (r) @@ -4157,8 +4185,6 @@ static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) iounmap(adev->rmmio); adev->rmmio = NULL; - if (adev->mman.aper_base_kaddr) - iounmap(adev->mman.aper_base_kaddr); adev->mman.aper_base_kaddr = NULL; /* Memory manager related */ @@ -4213,7 +4239,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) /* disable ras feature must before hw fini */ amdgpu_ras_pre_fini(adev); - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_ttm_disable_buffer_funcs(adev); /* * device went through surprise hotplug; we need to destroy topology @@ -4480,7 +4506,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients) if (r) goto unwind_userq; - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_ttm_disable_buffer_funcs(adev); amdgpu_fence_driver_hw_fini(adev); @@ -4494,7 +4520,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients) return 0; unwind_evict: - amdgpu_ttm_set_buffer_funcs_status(adev, true); + amdgpu_ttm_enable_buffer_funcs(adev); amdgpu_fence_driver_hw_init(adev); unwind_userq: @@ -5087,11 +5113,12 @@ link_reset_failed: int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, struct amdgpu_reset_context *reset_context) { - int i, r = 0; struct amdgpu_job *job = NULL; + struct dma_fence *fence = NULL; struct amdgpu_device *tmp_adev = reset_context->reset_req_dev; bool need_full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); + int i, r; if (reset_context->reset_req_dev == adev) job = reset_context->job; @@ -5101,6 +5128,9 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, amdgpu_fence_driver_isr_toggle(adev, true); + if (job) + fence = &job->hw_fence->base; + /* block all schedulers and reset given job's ring */ for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring = adev->rings[i]; @@ -5109,14 +5139,11 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, continue; /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ - amdgpu_fence_driver_force_completion(ring); + amdgpu_fence_driver_force_completion(ring, fence); } amdgpu_fence_driver_isr_toggle(adev, false); - if (job && job->vm) - drm_sched_increase_karma(&job->base); - r = amdgpu_reset_prepare_hwcontext(adev, reset_context); /* If reset handler not implemented, continue; otherwise return */ if (r == -EOPNOTSUPP) @@ -5228,7 +5255,7 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context) if (r) goto out; - amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true); + amdgpu_ttm_enable_buffer_funcs(tmp_adev); r = amdgpu_device_ip_resume_phase3(tmp_adev); if (r) @@ -5914,8 +5941,6 @@ static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev, enum pci_bus_speed *speed, enum pcie_link_width *width) { - struct pci_dev *parent = adev->pdev; - if (!speed || !width) return; @@ -5923,13 +5948,11 @@ static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev, *width = PCIE_LNK_WIDTH_UNKNOWN; if (amdgpu_device_pcie_dynamic_switching_supported(adev)) { - while ((parent = pci_upstream_bridge(parent))) { - /* skip upstream/downstream switches internal to dGPU*/ - if (parent->vendor == PCI_VENDOR_ID_ATI) - continue; + struct pci_dev *parent = amdgpu_device_find_parent(adev); + + if (parent) { *speed = pcie_get_speed_cap(parent); *width = pcie_get_width_cap(parent); - break; } } else { /* use the current speeds rather than max if switching is not supported */ @@ -5956,22 +5979,15 @@ static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev, if (!speed || !width) return; - parent = pci_upstream_bridge(parent); - if (parent && parent->vendor == PCI_VENDOR_ID_ATI) { - /* use the upstream/downstream switches internal to dGPU */ + /* use the device itself */ + *speed = pcie_get_speed_cap(adev->pdev); + *width = pcie_get_width_cap(adev->pdev); + + /* use the link outside the device */ + parent = amdgpu_device_find_parent(adev); + if (parent) { *speed = pcie_get_speed_cap(parent); *width = pcie_get_width_cap(parent); - while ((parent = pci_upstream_bridge(parent))) { - if (parent->vendor == PCI_VENDOR_ID_ATI) { - /* use the upstream/downstream switches internal to dGPU */ - *speed = pcie_get_speed_cap(parent); - *width = pcie_get_width_cap(parent); - } - } - } else { - /* use the device itself */ - *speed = pcie_get_speed_cap(adev->pdev); - *width = pcie_get_width_cap(adev->pdev); } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index 80efeca0ab73..7b9bb998906d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -64,6 +64,7 @@ #include "nbio_v7_2.h" #include "nbio_v7_7.h" #include "nbif_v6_3_1.h" +#include "nbio_v6_3_2.h" #include "hdp_v5_0.h" #include "hdp_v5_2.h" #include "hdp_v6_0.h" @@ -395,6 +396,26 @@ static int amdgpu_discovery_read_binary_from_file(struct amdgpu_device *adev, return r; } + if (fw->size > adev->discovery.size) { + dev_err(adev->dev, + "ip discovery firmware \"%s\" too large (%zu > %u)\n", + fw_name, fw->size, adev->discovery.size); + release_firmware(fw); + return -EINVAL; + } + + /* Ensure the firmware is at least large enough to contain the + * binary header fields. + */ + if (fw->size < offsetof(struct binary_header, binary_size) + + sizeof(((struct binary_header *)0)->binary_size)) { + dev_err(adev->dev, + "ip discovery firmware \"%s\" too small (%zu)\n", + fw_name, fw->size); + release_firmware(fw); + return -EINVAL; + } + memcpy((u8 *)binary, (u8 *)fw->data, fw->size); release_firmware(fw); @@ -819,42 +840,44 @@ static void amdgpu_discovery_read_from_harvest_table(struct amdgpu_device *adev, harvest_info = (struct harvest_table *)(discovery_bin + offset); for (i = 0; i < 32; i++) { - if (le16_to_cpu(harvest_info->list[i].hw_id) == 0) + u16 hw_id = le16_to_cpu(harvest_info->list[i].hw_id); + u8 inst = harvest_info->list[i].number_instance; + + if (hw_id == 0) break; - switch (le16_to_cpu(harvest_info->list[i].hw_id)) { + if (inst >= 32) { + dev_warn(adev->dev, + "bogus harvest instance %u for hw_id %u\n", + inst, hw_id); + continue; + } + + switch (hw_id) { case VCN_HWID: (*vcn_harvest_count)++; - adev->vcn.harvest_config |= - (1 << harvest_info->list[i].number_instance); - adev->jpeg.harvest_config |= - (1 << harvest_info->list[i].number_instance); - - adev->vcn.inst_mask &= - ~(1U << harvest_info->list[i].number_instance); - adev->jpeg.inst_mask &= - ~(1U << harvest_info->list[i].number_instance); + adev->vcn.harvest_config |= BIT(inst); + adev->jpeg.harvest_config |= BIT(inst); + + adev->vcn.inst_mask &= ~BIT(inst); + adev->jpeg.inst_mask &= ~BIT(inst); break; case DMU_HWID: adev->harvest_ip_mask |= AMD_HARVEST_IP_DMU_MASK; break; case UMC_HWID: - umc_harvest_config |= - 1 << (le16_to_cpu(harvest_info->list[i].number_instance)); + umc_harvest_config |= BIT_ULL(inst); (*umc_harvest_count)++; break; case GC_HWID: - adev->gfx.xcc_mask &= - ~(1U << harvest_info->list[i].number_instance); + adev->gfx.xcc_mask &= ~BIT(inst); break; case SDMA0_HWID: - adev->sdma.sdma_mask &= - ~(1U << harvest_info->list[i].number_instance); + adev->sdma.sdma_mask &= ~BIT(inst); break; #if defined(CONFIG_DRM_AMD_ISP) case ISP_HWID: - adev->isp.harvest_config |= - ~(1U << harvest_info->list[i].number_instance); + adev->isp.harvest_config |= ~BIT(inst); break; #endif default: @@ -2095,6 +2118,9 @@ static int amdgpu_discovery_set_common_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): amdgpu_device_ip_block_add(adev, &soc21_common_ip_block); break; case IP_VERSION(12, 0, 0): @@ -2155,6 +2181,9 @@ static int amdgpu_discovery_set_gmc_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): amdgpu_device_ip_block_add(adev, &gmc_v11_0_ip_block); break; case IP_VERSION(12, 0, 0): @@ -2202,6 +2231,7 @@ static int amdgpu_discovery_set_ih_ip_blocks(struct amdgpu_device *adev) break; case IP_VERSION(6, 1, 0): case IP_VERSION(6, 1, 1): + case IP_VERSION(6, 4, 0): amdgpu_device_ip_block_add(adev, &ih_v6_1_ip_block); break; case IP_VERSION(7, 0, 0): @@ -2274,6 +2304,7 @@ static int amdgpu_discovery_set_psp_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &psp_v14_0_ip_block); break; case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 9): amdgpu_device_ip_block_add(adev, &psp_v15_0_ip_block); break; case IP_VERSION(15, 0, 8): @@ -2343,7 +2374,9 @@ static int amdgpu_discovery_set_smu_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &smu_v14_0_ip_block); break; case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 5): case IP_VERSION(15, 0, 8): + case IP_VERSION(15, 0, 9): amdgpu_device_ip_block_add(adev, &smu_v15_0_ip_block); break; default: @@ -2398,6 +2431,7 @@ static int amdgpu_discovery_set_display_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(3, 6, 0): case IP_VERSION(4, 1, 0): case IP_VERSION(4, 2, 0): + case IP_VERSION(4, 2, 1): /* TODO: Fix IP version. DC code expects version 4.0.1 */ if (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(4, 1, 0)) adev->ip_versions[DCE_HWIP][0] = IP_VERSION(4, 0, 1); @@ -2477,6 +2511,9 @@ static int amdgpu_discovery_set_gc_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): amdgpu_device_ip_block_add(adev, &gfx_v11_0_ip_block); break; case IP_VERSION(12, 0, 0): @@ -2537,6 +2574,7 @@ static int amdgpu_discovery_set_sdma_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(6, 1, 2): case IP_VERSION(6, 1, 3): case IP_VERSION(6, 1, 4): + case IP_VERSION(6, 4, 0): amdgpu_device_ip_block_add(adev, &sdma_v6_0_ip_block); break; case IP_VERSION(7, 0, 0): @@ -2688,6 +2726,9 @@ static int amdgpu_discovery_set_mes_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): amdgpu_device_ip_block_add(adev, &mes_v11_0_ip_block); adev->enable_mes = true; adev->enable_mes_kiq = true; @@ -2737,6 +2778,10 @@ static int amdgpu_discovery_set_vpe_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(6, 1, 3): amdgpu_device_ip_block_add(adev, &vpe_v6_1_ip_block); break; + case IP_VERSION(2, 0, 0): + case IP_VERSION(2, 2, 0): + amdgpu_device_ip_block_add(adev, &vpe_v2_0_ip_block); + break; default: break; } @@ -3091,6 +3136,9 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->family = AMDGPU_FAMILY_GC_11_5_0; break; case IP_VERSION(12, 0, 0): @@ -3119,6 +3167,9 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->flags |= AMD_IS_APU; break; default: @@ -3190,9 +3241,13 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) break; case IP_VERSION(6, 3, 1): case IP_VERSION(7, 11, 4): + case IP_VERSION(7, 11, 5): adev->nbio.funcs = &nbif_v6_3_1_funcs; adev->nbio.hdp_flush_reg = &nbif_v6_3_1_hdp_flush_reg; break; + case IP_VERSION(6, 3, 2): + adev->nbio.funcs = &nbio_v6_3_2_funcs; + break; default: break; } @@ -3225,6 +3280,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(6, 0, 1): case IP_VERSION(6, 1, 0): case IP_VERSION(6, 1, 1): + case IP_VERSION(6, 4, 0): adev->hdp.funcs = &hdp_v6_0_funcs; break; case IP_VERSION(7, 0, 0): @@ -3307,6 +3363,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) adev->smuio.funcs = &smuio_v14_0_2_funcs; break; case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 5): adev->smuio.funcs = &smuio_v15_0_0_funcs; break; case IP_VERSION(15, 0, 8): @@ -3416,3 +3473,28 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) return 0; } +int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev, + uint16_t *major, uint16_t *minor) +{ + uint8_t *discovery_bin = adev->discovery.bin; + struct table_info *info; + union gc_info *gc_info; + u16 offset; + + if (!discovery_bin) + return -EINVAL; + if (amdgpu_discovery_get_table_info(adev, &info, GC)) + return -EINVAL; + + offset = le16_to_cpu(info->offset); + if (!offset) + return -EINVAL; + + gc_info = (union gc_info *)(discovery_bin + offset); + + if (major) + *major = le16_to_cpu(gc_info->v1.header.version_major); + if (minor) + *minor = le16_to_cpu(gc_info->v1.header.version_minor); + return 0; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h index 0ff1a7923eed..e0010f6a3eda 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h @@ -48,6 +48,8 @@ int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev, uint32_t *nps_type, struct amdgpu_gmc_memrange *ranges, int *range_cnt, bool refresh); +int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev, + uint16_t *major, uint16_t *minor); void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 60debd543e44..4c0c77eafbd1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -246,6 +246,7 @@ int amdgpu_umsch_mm_fwlog; int amdgpu_rebar = -1; /* auto */ int amdgpu_user_queue = -1; uint amdgpu_hdmi_hpd_debounce_delay_ms; +int amdgpu_ptl = -1; /* auto */ DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, "DRM_UT_CORE", @@ -1112,6 +1113,18 @@ module_param_named(user_queue, amdgpu_user_queue, int, 0444); MODULE_PARM_DESC(hdmi_hpd_debounce_delay_ms, "HDMI HPD disconnect debounce delay in milliseconds (0 to disable (by default), 1500 is common)"); module_param_named(hdmi_hpd_debounce_delay_ms, amdgpu_hdmi_hpd_debounce_delay_ms, uint, 0644); +/** + * DOC: ptl (int) + * Enable PTL feature at boot time. Possible values: + * + * - -1 = auto (ASIC specific default) + * - 0 = disable PTL (default) + * - 1 = enable PTL + * - 2 = permanently disable PTL (cannot be re-enabled at runtime) + */ +MODULE_PARM_DESC(ptl, "Enable PTL (-1 = auto, 0 = disable (default), 1 = enable, 2 = permanently disable)"); +module_param_named(ptl, amdgpu_ptl, int, 0444); + /* These devices are not supported by amdgpu. * They are supported by the mach64, r128, radeon drivers */ @@ -2797,12 +2810,11 @@ static int amdgpu_runtime_idle_check_userq(struct device *dev) return xa_empty(&adev->userq_doorbell_xa) ? 0 : -EBUSY; } -static int amdgpu_pmops_runtime_suspend(struct device *dev) +static int amdgpu_pmops_runtime_checks(struct device *dev) { - struct pci_dev *pdev = to_pci_dev(dev); - struct drm_device *drm_dev = pci_get_drvdata(pdev); + struct drm_device *drm_dev = dev_get_drvdata(dev); struct amdgpu_device *adev = drm_to_adev(drm_dev); - int ret, i; + int ret; if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) { pm_runtime_forbid(dev); @@ -2812,7 +2824,27 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) ret = amdgpu_runtime_idle_check_display(dev); if (ret) return ret; - ret = amdgpu_runtime_idle_check_userq(dev); + + return amdgpu_runtime_idle_check_userq(dev); +} + +static int amdgpu_pmops_runtime_idle(struct device *dev) +{ + int ret; + + ret = amdgpu_pmops_runtime_checks(dev); + pm_runtime_autosuspend(dev); + return ret; +} + +static int amdgpu_pmops_runtime_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + struct amdgpu_device *adev = drm_to_adev(drm_dev); + int ret, i; + + ret = amdgpu_pmops_runtime_checks(dev); if (ret) return ret; @@ -2924,27 +2956,6 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) return 0; } -static int amdgpu_pmops_runtime_idle(struct device *dev) -{ - struct drm_device *drm_dev = dev_get_drvdata(dev); - struct amdgpu_device *adev = drm_to_adev(drm_dev); - int ret; - - if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) { - pm_runtime_forbid(dev); - return -EBUSY; - } - - ret = amdgpu_runtime_idle_check_display(dev); - if (ret) - goto done; - - ret = amdgpu_runtime_idle_check_userq(dev); -done: - pm_runtime_autosuspend(dev); - return ret; -} - static int amdgpu_drm_release(struct inode *inode, struct file *filp) { struct drm_file *file_priv = filp->private_data; @@ -3147,6 +3158,9 @@ static int __init amdgpu_init(void) { int r; + /* Train lockdep on correct lock ordering */ + amdgpu_lockdep_init(); + r = amdgpu_sync_init(); if (r) return r; @@ -3182,6 +3196,14 @@ static void __exit amdgpu_exit(void) amdgpu_sync_fini(); mmu_notifier_synchronize(); amdgpu_xcp_drv_release(); + + /* + * Flush outstanding call_rcu() callbacks before the + * module text is freed. Otherwise a grace period elapsing after + * unload invokes a callback in already-freed module memory and + * faults in rcu_do_batch(). + */ + rcu_barrier(); } module_init(amdgpu_init); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 4c5e38dea4c2..f6b7522c3c82 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -121,7 +121,6 @@ int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr, { struct amdgpu_eviction_fence *ev_fence; struct drm_gem_object *obj; - unsigned long index; /* Create and initialize a new eviction fence */ ev_fence = kzalloc_obj(*ev_fence); @@ -140,7 +139,7 @@ int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr, evf_mgr->ev_fence = &ev_fence->base; /* And add it to all existing BOs */ - drm_exec_for_each_locked_object(exec, index, obj) { + drm_exec_for_each_locked_object(exec, obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); amdgpu_evf_mgr_attach_fence(evf_mgr, bo); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 8048a4c04b47..ea69b1bac7c6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -547,7 +547,7 @@ void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev) r = -ENODEV; /* no need to trigger GPU reset as we are unloading */ if (r) - amdgpu_fence_driver_force_completion(ring); + amdgpu_fence_driver_force_completion(ring, NULL); if (!drm_dev_is_unplugged(adev_to_drm(adev)) && ring->fence_drv.irq_src && @@ -662,16 +662,34 @@ void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error) * amdgpu_fence_driver_force_completion - force signal latest fence of ring * * @ring: fence of the ring to signal + * @timedout_fence: fence of the timedout job * */ -void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring) +void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring, + struct dma_fence *timedout_fence) { - amdgpu_fence_driver_set_error(ring, -ECANCELED); + struct amdgpu_fence_driver *drv = &ring->fence_drv; + unsigned long flags; + + spin_lock_irqsave(&drv->lock, flags); + for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) { + struct dma_fence *fence; + + fence = rcu_dereference_protected(drv->fences[i], + lockdep_is_held(&drv->lock)); + if (fence && !dma_fence_is_signaled_locked(fence)) { + if (fence == timedout_fence) + dma_fence_set_error(fence, -ETIME); + else + dma_fence_set_error(fence, -ECANCELED); + } + } + spin_unlock_irqrestore(&drv->lock, flags); + amdgpu_fence_write(ring, ring->fence_drv.sync_seq); amdgpu_fence_process(ring); } - /* * Kernel queue reset handling * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c index 841e1b3a017e..18078844ddc7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c @@ -115,6 +115,43 @@ static bool is_fru_eeprom_supported(struct amdgpu_device *adev, u32 *fru_addr) } } +/* + * IPMI FRU Product Info Area fields are TLV: one type/length byte + * (low 6 bits = data length) followed by that many data bytes. These + * helpers walk the cursor and copy a single field while bounding all + * accesses to the actual buffer length read from the EEPROM. + */ +#define FRU_FIELD_LEN(p, a) ((p)[a] & 0x3F) + +/* Advance cursor past the current TLV. Returns false if no more data. */ +static bool fru_pia_advance(u32 *addr, const unsigned char *pia, int len) +{ + if (*addr >= (u32)len) + return false; + *addr += 1 + FRU_FIELD_LEN(pia, *addr); + return true; +} + +/* + * Copy the current TLV's data into dst (NUL-terminated). Returns false if + * the TLV header or data would read past the end of pia. + */ +static bool fru_pia_copy_field(char *dst, size_t dst_size, + const unsigned char *pia, u32 addr, int len) +{ + size_t fl; + + if (addr + 1 >= (u32)len) + return false; + + fl = min3((size_t)FRU_FIELD_LEN(pia, addr), + dst_size - 1, + (size_t)(len - addr - 1)); + memcpy(dst, pia + addr + 1, fl); + dst[fl] = '\0'; + return true; +} + int amdgpu_fru_get_product_info(struct amdgpu_device *adev) { struct amdgpu_fru_info *fru_info; @@ -223,52 +260,46 @@ int amdgpu_fru_get_product_info(struct amdgpu_device *adev) * Read Manufacturer Name field whose length is [3]. */ addr = 3; - if (addr + 1 >= len) + if (!fru_pia_copy_field(fru_info->manufacturer_name, + sizeof(fru_info->manufacturer_name), + pia, addr, len)) goto Out; - memcpy(fru_info->manufacturer_name, pia + addr + 1, - min_t(size_t, sizeof(fru_info->manufacturer_name), - pia[addr] & 0x3F)); - fru_info->manufacturer_name[sizeof(fru_info->manufacturer_name) - 1] = - '\0'; /* Read Product Name field. */ - addr += 1 + (pia[addr] & 0x3F); - if (addr + 1 >= len) + if (!fru_pia_advance(&addr, pia, len) || + !fru_pia_copy_field(fru_info->product_name, + sizeof(fru_info->product_name), + pia, addr, len)) goto Out; - memcpy(fru_info->product_name, pia + addr + 1, - min_t(size_t, sizeof(fru_info->product_name), pia[addr] & 0x3F)); - fru_info->product_name[sizeof(fru_info->product_name) - 1] = '\0'; /* Go to the Product Part/Model Number field. */ - addr += 1 + (pia[addr] & 0x3F); - if (addr + 1 >= len) + if (!fru_pia_advance(&addr, pia, len) || + !fru_pia_copy_field(fru_info->product_number, + sizeof(fru_info->product_number), + pia, addr, len)) goto Out; - memcpy(fru_info->product_number, pia + addr + 1, - min_t(size_t, sizeof(fru_info->product_number), - pia[addr] & 0x3F)); - fru_info->product_number[sizeof(fru_info->product_number) - 1] = '\0'; - /* Go to the Product Version field. */ - addr += 1 + (pia[addr] & 0x3F); + /* Skip the Product Version field. */ + if (!fru_pia_advance(&addr, pia, len)) + goto Out; - /* Go to the Product Serial Number field. */ - addr += 1 + (pia[addr] & 0x3F); - if (addr + 1 >= len) + /* Read the Product Serial Number field. */ + if (!fru_pia_advance(&addr, pia, len) || + !fru_pia_copy_field(fru_info->serial, + sizeof(fru_info->serial), + pia, addr, len)) goto Out; - memcpy(fru_info->serial, pia + addr + 1, - min_t(size_t, sizeof(fru_info->serial), pia[addr] & 0x3F)); - fru_info->serial[sizeof(fru_info->serial) - 1] = '\0'; - /* Asset Tag field */ - addr += 1 + (pia[addr] & 0x3F); + /* Skip the Asset Tag field. */ + if (!fru_pia_advance(&addr, pia, len)) + goto Out; /* FRU File Id field. This could be 'null'. */ - addr += 1 + (pia[addr] & 0x3F); - if ((addr + 1 >= len) || !(pia[addr] & 0x3F)) + if (!fru_pia_advance(&addr, pia, len) || + !fru_pia_copy_field(fru_info->fru_id, + sizeof(fru_info->fru_id), + pia, addr, len)) goto Out; - memcpy(fru_info->fru_id, pia + addr + 1, - min_t(size_t, sizeof(fru_info->fru_id), pia[addr] & 0x3F)); - fru_info->fru_id[sizeof(fru_info->fru_id) - 1] = '\0'; Out: kfree(pia); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index fe6d988e7f24..6a0699746fbc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -27,6 +27,7 @@ */ #include <linux/ktime.h> #include <linux/module.h> +#include <linux/overflow.h> #include <linux/pagemap.h> #include <linux/pci.h> #include <linux/dma-buf.h> @@ -248,7 +249,7 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj, drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); drm_exec_until_all_locked(&exec) { - r = drm_exec_prepare_obj(&exec, &abo->tbo.base, 1); + r = drm_exec_prepare_obj(&exec, &abo->tbo.base, TTM_NUM_MOVE_FENCES + 1); drm_exec_retry_on_contention(&exec); if (unlikely(r)) goto out_unlock; @@ -534,6 +535,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, bo = gem_to_amdgpu_bo(gobj); bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; + bo->parent = amdgpu_bo_ref(fpriv->vm.root.bo); r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags); if (r) goto release_object; @@ -1093,6 +1095,11 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, * If that number is larger than the size of the array, the ioctl must * be retried. */ + if (!bo_va) { + r = -ENOENT; + goto out_exec; + } + if (args->num_entries > INT_MAX / sizeof(*vm_entries)) { r = -EINVAL; goto out_exec; @@ -1228,13 +1235,14 @@ int amdgpu_gem_list_handles_ioctl(struct drm_device *dev, void *data, return ret; } -static int amdgpu_gem_align_pitch(struct amdgpu_device *adev, - int width, - int cpp, - bool tiled) +static unsigned int amdgpu_gem_align_pitch(struct amdgpu_device *adev, + unsigned int width, + unsigned int cpp, + bool tiled) { - int aligned = width; - int pitch_mask = 0; + unsigned int aligned = width; + unsigned int pitch_mask = 0; + unsigned int pitch; switch (cpp) { case 1: @@ -1249,9 +1257,12 @@ static int amdgpu_gem_align_pitch(struct amdgpu_device *adev, break; } - aligned += pitch_mask; + if (check_add_overflow(aligned, pitch_mask, &aligned)) + return 0; aligned &= ~pitch_mask; - return aligned * cpp; + if (check_mul_overflow(aligned, cpp, &pitch)) + return 0; + return pitch; } int amdgpu_mode_dumb_create(struct drm_file *file_priv, @@ -1278,8 +1289,12 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, args->pitch = amdgpu_gem_align_pitch(adev, args->width, DIV_ROUND_UP(args->bpp, 8), 0); + if (!args->pitch) + return -EINVAL; args->size = (u64)args->pitch * args->height; args->size = ALIGN(args->size, PAGE_SIZE); + if (!args->size) + return -EINVAL; domain = amdgpu_bo_get_preferred_domain(adev, amdgpu_display_supported_domains(adev, flags)); r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index b8ca876694ff..85372af1216d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -990,10 +990,7 @@ int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *r if (r) return r; - if (amdgpu_sriov_vf(adev)) - return r; - - if (adev->gfx.cp_ecc_error_irq.funcs) { + if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs) { r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); if (r) goto late_fini; @@ -1008,6 +1005,21 @@ late_fini: return r; } +void amdgpu_gfx_ras_suspend(struct amdgpu_device *adev, + struct ras_common_if *ras_block) +{ + if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs) + amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); +} + +void amdgpu_gfx_ras_fini(struct amdgpu_device *adev, + struct ras_common_if *ras_block) +{ + if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs) + amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); + amdgpu_ras_block_late_fini(adev, ras_block); +} + int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev) { int err = 0; @@ -1036,6 +1048,12 @@ int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev) if (!ras->ras_block.ras_late_init) ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init; + if (!ras->ras_block.ras_suspend) + ras->ras_block.ras_suspend = amdgpu_gfx_ras_suspend; + + if (!ras->ras_block.ras_fini) + ras->ras_block.ras_fini = amdgpu_gfx_ras_fini; + /* If not defined special ras_cb function, use default ras_cb */ if (!ras->ras_block.ras_cb) ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb; @@ -1646,12 +1664,13 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) struct amdgpu_device *adev = ring->adev; struct drm_gpu_scheduler *sched = &ring->sched; struct drm_sched_entity entity; + unsigned int ib_size_dw = 16; static atomic_t counter; struct dma_fence *f; struct amdgpu_job *job; struct amdgpu_ib *ib; void *owner; - int i, r; + int r; /* Initialize the scheduler entity */ r = drm_sched_entity_init(&entity, DRM_SCHED_PRIORITY_NORMAL, @@ -1669,7 +1688,7 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) owner = (void *)(unsigned long)atomic_inc_return(&counter); r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner, - 64, 0, &job, + ib_size_dw * sizeof(uint32_t), 0, &job, AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER); if (r) goto err; @@ -1679,9 +1698,8 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) job->run_cleaner_shader = true; ib = &job->ibs[0]; - for (i = 0; i <= ring->funcs->align_mask; ++i) - ib->ptr[i] = ring->funcs->nop; - ib->length_dw = ring->funcs->align_mask + 1; + memset32(ib->ptr, ring->funcs->nop, ib_size_dw); + ib->length_dw = ib_size_dw; f = amdgpu_job_submit(job); @@ -2686,3 +2704,54 @@ void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev) #endif } +int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; + struct amdgpu_ring *kiq_ring = &kiq->ring; + unsigned long flags; + int i; + + if (adev->enable_mes) + return -EINVAL; + + if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) + return -EINVAL; + + spin_lock_irqsave(&kiq->ring_lock, flags); + + if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { + spin_unlock_irqrestore(&kiq->ring_lock, flags); + return -ENOMEM; + } + + /* assert preemption condition */ + amdgpu_ring_set_preempt_cond_exec(ring, false); + + /* assert IB preemption, emit the trailing fence */ + kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, + ring->trail_fence_gpu_addr, + ++ring->trail_seq); + amdgpu_ring_commit(kiq_ring); + + spin_unlock_irqrestore(&kiq->ring_lock, flags); + + /* poll the trailing fence */ + for (i = 0; i < adev->usec_timeout; i++) { + if (ring->trail_seq == + le32_to_cpu(*(ring->trail_fence_cpu_addr))) + break; + udelay(1); + } + + /* deassert preemption condition */ + amdgpu_ring_set_preempt_cond_exec(ring, true); + + if (i >= adev->usec_timeout) { + DRM_ERROR("ring %d failed to preempt ib\n", ring->idx); + return -EINVAL; + } + + return 0; +} + diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index a0cf0a3b41da..54c1eb9c499b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -615,7 +615,8 @@ void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable); void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable); int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value); int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block); -void amdgpu_gfx_ras_fini(struct amdgpu_device *adev); +void amdgpu_gfx_ras_suspend(struct amdgpu_device *adev, struct ras_common_if *ras_block); +void amdgpu_gfx_ras_fini(struct amdgpu_device *adev, struct ras_common_if *ras_block); int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value); int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *residency); int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value); @@ -664,6 +665,8 @@ void amdgpu_gfx_csb_preamble_end(u32 *buffer, u32 count); void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev); void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev); +int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring); + static inline const char *amdgpu_gfx_compute_mode_desc(int mode) { switch (mode) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index c076c5f06e77..5d6149ba7ab7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -280,6 +280,15 @@ void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc mc->gart_size >> 20, mc->gart_start, mc->gart_end); } +void amdgpu_gmc_set_gart_size(struct amdgpu_device *adev, u64 default_size) +{ + if (amdgpu_gart_size == -1) + adev->gmc.gart_size = + default_size + adev->pm.smu_prv_buffer_size; + else + adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; +} + /** * amdgpu_gmc_gart_location - try to find GART location * @@ -711,12 +720,14 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev) void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, uint32_t vmhub, uint32_t flush_type) { - struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + struct amdgpu_ring *ring; struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; struct dma_fence *fence; struct amdgpu_job *job; int r; + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); + if (!hub->sdma_invalidation_workaround || vmid || !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready || !ring->sched.ready) { @@ -965,6 +976,9 @@ void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): /* Don't enable it by default yet. */ if (amdgpu_tmz < 1) { @@ -1005,6 +1019,9 @@ void amdgpu_gmc_noretry_set(struct amdgpu_device *adev) gc_ver == IP_VERSION(9, 5, 0) || gc_ver >= IP_VERSION(10, 1, 0)); + /* For GFX12.1 B0, set xnack (retry) on as default */ + if (gc_ver == IP_VERSION(12, 1, 0) && (adev->rev_id & 0xf) == 0x1) + noretry_default = false; if (!amdgpu_sriov_xnack_support(adev)) gmc->noretry = 1; else diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h index d03536b969b5..ddb0d500e0fa 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h @@ -366,6 +366,8 @@ struct amdgpu_gmc { bool flush_tlb_needs_extra_type_0; bool flush_tlb_needs_extra_type_2; bool flush_pasid_uses_kiq; + + bool override_pte; }; #define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr)) @@ -485,4 +487,6 @@ void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev, struct amdgpu_mem_partition_info *mem_ranges); int amdgpu_gmc_get_vram_info(struct amdgpu_device *adev, int *vram_width, int *vram_type, int *vram_vendor); + +void amdgpu_gmc_set_gart_size(struct amdgpu_device *adev, u64 default_size); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c index d23a91d029aa..0ea32561c4bc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c @@ -272,7 +272,20 @@ static bool amdgpu_gtt_mgr_intersects(struct ttm_resource_manager *man, const struct ttm_place *place, size_t size) { - return !place->lpfn || amdgpu_gtt_mgr_has_gart_addr(res); + const struct drm_mm_node *const node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + const u32 num_pages = PFN_UP(size); + + if (!place->lpfn) + return true; + + if (!amdgpu_gtt_mgr_has_gart_addr(res)) + return false; + + if (place->fpfn >= (node->start + num_pages) || + (place->lpfn && place->lpfn <= node->start)) + return false; + + return true; } /** @@ -290,7 +303,20 @@ static bool amdgpu_gtt_mgr_compatible(struct ttm_resource_manager *man, const struct ttm_place *place, size_t size) { - return !place->lpfn || amdgpu_gtt_mgr_has_gart_addr(res); + const struct drm_mm_node *const node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + const u32 num_pages = PFN_UP(size); + + if (!place->lpfn) + return true; + + if (!amdgpu_gtt_mgr_has_gart_addr(res)) + return false; + + if (node->start < place->fpfn || + (place->lpfn && (node->start + num_pages) > place->lpfn)) + return false; + + return true; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c index e452444b33b0..a7d13e337d84 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c @@ -67,7 +67,6 @@ static bool amdgpu_hmm_invalidate_gfx(struct mmu_interval_notifier *mni, { struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier); struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); - struct amdgpu_bo *vm_root = bo->vm_bo->vm->root.bo; long r; if (!mmu_notifier_range_blockable(range)) @@ -78,7 +77,7 @@ static bool amdgpu_hmm_invalidate_gfx(struct mmu_interval_notifier *mni, mmu_interval_set_seq(mni, cur_seq); amdgpu_vm_bo_invalidate(bo, false); - r = dma_resv_wait_timeout(vm_root->tbo.base.resv, + r = dma_resv_wait_timeout(bo->parent->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP, false, MAX_SCHEDULE_TIMEOUT); mutex_unlock(&adev->notifier_lock); @@ -174,7 +173,6 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, const u64 max_bytes = SZ_2G; struct hmm_range *hmm_range = &range->hmm_range; - unsigned long timeout; unsigned long *pfns; unsigned long end; int r; @@ -201,15 +199,9 @@ int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, pr_debug("hmm range: start = 0x%lx, end = 0x%lx", hmm_range->start, hmm_range->end); - timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); - -retry: r = hmm_range_fault(hmm_range); - if (unlikely(r)) { - if (r == -EBUSY && !time_after(jiffies, timeout)) - goto retry; + if (unlikely(r)) goto out_free_pfns; - } if (hmm_range->end == end) break; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h index f58b6be7fccc..444437c30088 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h @@ -91,6 +91,12 @@ struct amdgpu_ih_funcs { uint64_t (*decode_iv_ts)(struct amdgpu_ih_ring *ih, u32 rptr, signed int offset); void (*set_rptr)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); + /* Decode IH cookie node_id into a human-readable die name string. + * Returns buf, or NULL if this IH version does not support node_id decoding. + */ + const char *(*node_id_to_die_name)(struct amdgpu_device *adev, + unsigned int node_id, + char *buf, size_t size); }; #define amdgpu_ih_get_wptr(adev, ih) (adev)->irq.ih_funcs->get_wptr((adev), (ih)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h index 9c56be725ff3..eb8537558f5f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h @@ -39,7 +39,6 @@ struct amdgpu_imu_funcs { int (*switch_compute_partition)(struct amdgpu_device *adev, int num_xccs_per_xcp, int compute_partition_mode); - void (*init_mcm_addr_lut)(struct amdgpu_device *adev); }; struct imu_rlc_ram_golden { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 254a4e983f40..53be764968e4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -309,8 +309,6 @@ int amdgpu_irq_init(struct amdgpu_device *adev) unsigned int irq, flags; int r; - spin_lock_init(&adev->irq.lock); - /* Enable MSI if not disabled by module parameter */ adev->irq.msi_enabled = false; @@ -547,7 +545,7 @@ void amdgpu_irq_delegate(struct amdgpu_device *adev, unsigned int num_dw) { amdgpu_ih_ring_write(adev, &adev->irq.ih_soft, entry->iv_entry, num_dw); - schedule_work(&adev->irq.ih_soft_work); + queue_work(system_unbound_wq, &adev->irq.ih_soft_work); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 07771721af9d..9ecc6387c1eb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -388,7 +388,9 @@ static struct dma_fence * amdgpu_job_prepare_job(struct drm_sched_job *sched_job, struct drm_sched_entity *s_entity) { - struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->rq->sched); + struct drm_gpu_scheduler *sched = + container_of(s_entity->rq, typeof(*sched), rq); + struct amdgpu_ring *ring = to_amdgpu_ring(sched); struct amdgpu_job *job = to_amdgpu_job(sched_job); struct dma_fence *fence; int r; @@ -481,25 +483,22 @@ drm_sched_entity_queue_pop(struct drm_sched_entity *entity) void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) { + struct drm_sched_rq *rq = &sched->rq; + struct drm_sched_entity *s_entity; struct drm_sched_job *s_job; - struct drm_sched_entity *s_entity = NULL; - int i; /* Signal all jobs not yet scheduled */ - for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { - struct drm_sched_rq *rq = sched->sched_rq[i]; - spin_lock(&rq->lock); - list_for_each_entry(s_entity, &rq->entities, list) { - while ((s_job = drm_sched_entity_queue_pop(s_entity))) { - struct drm_sched_fence *s_fence = s_job->s_fence; - - dma_fence_signal(&s_fence->scheduled); - dma_fence_set_error(&s_fence->finished, -EHWPOISON); - dma_fence_signal(&s_fence->finished); - } + spin_lock(&rq->lock); + list_for_each_entry(s_entity, &rq->entities, list) { + while ((s_job = drm_sched_entity_queue_pop(s_entity))) { + struct drm_sched_fence *s_fence = s_job->s_fence; + + dma_fence_signal(&s_fence->scheduled); + dma_fence_set_error(&s_fence->finished, -EHWPOISON); + dma_fence_signal(&s_fence->finished); } - spin_unlock(&rq->lock); } + spin_unlock(&rq->lock); /* Signal all jobs already scheduled to HW */ list_for_each_entry(s_job, &sched->pending_list, list) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index 56a88e14a044..e70a1117b812 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h @@ -107,7 +107,10 @@ struct amdgpu_job { static inline struct amdgpu_ring *amdgpu_job_ring(struct amdgpu_job *job) { - return to_amdgpu_ring(job->base.entity->rq->sched); + struct drm_gpu_scheduler *sched = + container_of(job->base.entity->rq, typeof(*sched), rq); + + return to_amdgpu_ring(sched); } int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c new file mode 100644 index 000000000000..61450af539a6 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2024 Advanced Micro Devices, Inc. + * + * Lockdep annotation for AMDGPU lock ordering + * + * This module teaches lockdep the correct lock ordering to catch + * potential deadlocks at development time rather than runtime. + * + * Based on dma-resv lockdep approach from: + * drivers/dma-buf/dma-resv.c:dma_resv_lockdep() + */ + +#include "amdgpu.h" +#include "amdgpu_reset.h" + +#ifdef CONFIG_LOCKDEP + +struct amdgpu_lockdep_dummy_locks { + struct mutex reset_lock; + struct mutex userq_sch_mutex; + struct mutex userq_mutex; + struct mutex notifier_lock; + struct mutex vram_lock; + struct mutex srbm_mutex; + struct mutex grbm_idx_mutex; + spinlock_t mmio_idx_lock; +}; + +/* Lock class keys for associating with real driver locks */ +static struct lock_class_key amdgpu_userq_sch_mutex_key; +static struct lock_class_key amdgpu_userq_mutex_key; +static struct lock_class_key amdgpu_notifier_lock_key; +static struct lock_class_key amdgpu_vram_lock_key; +static struct lock_class_key amdgpu_reset_sem_key; +static struct lock_class_key amdgpu_reset_lock_key; +static struct lock_class_key amdgpu_srbm_lock_key; +static struct lock_class_key amdgpu_grbm_lock_key; +static struct lock_class_key amdgpu_mmio_lock_key; + +/** + * amdgpu_lockdep_set_class - Associate lock class keys with real locks + * @adev: AMDGPU device + * + * Call during device init to associate lock classes with actual locks + * so lockdep can track them properly. + */ +void amdgpu_lockdep_set_class(struct amdgpu_device *adev) +{ + lockdep_set_class(&adev->gfx.userq_sch_mutex, + &amdgpu_userq_sch_mutex_key); + lockdep_set_class(&adev->notifier_lock, &amdgpu_notifier_lock_key); + lockdep_set_class(&adev->srbm_mutex, &amdgpu_srbm_lock_key); + lockdep_set_class(&adev->grbm_idx_mutex, &amdgpu_grbm_lock_key); + lockdep_set_class(&adev->mmio_idx_lock, &amdgpu_mmio_lock_key); + + if (adev->reset_domain) + lockdep_set_class(&adev->reset_domain->sem, + &amdgpu_reset_sem_key); +} + +/** + * amdgpu_lockdep_init - Teach lockdep the correct lock ordering + * + * Instantiates dummy objects and takes locks in the correct order to + * train lockdep. This helps catch lock ordering violations during + * development. + * + * Lock ordering hierarchy (outermost to innermost): + * + * 1. userq_sch_mutex - Global userq scheduler (enforce_isolation) + * 2. userq_mutex - Per-context userq (held across queue create/destroy) + * 3. notifier_lock - MMU notifier lock + * 4. vram_lock - VRAM allocator lock + * 5. reset_domain->sem - GPU reset synchronization + * 6. reset_lock - Reset control lock + * 7. srbm_mutex - SRBM register access + * 8. grbm_idx_mutex - GRBM index access + * 9. mmio_idx_lock - MMIO index access (spinlock) + * + * Evidence: + * - userq_sch_mutex -> userq_mutex: amdgpu_gfx_kfd_sch_ctrl() calls + * amdgpu_userq_stop_sched_for_enforce_isolation() which takes userq_mutex + * - userq_mutex -> notifier_lock: userq paths may trigger MMU notifier + * invalidation which acquires notifier_lock + * - notifier_lock -> reset_domain->sem: HMM invalidation callback holds + * notifier_lock and can wait for GPU reset completion, so notifier_lock + * must be outer to reset_domain->sem + * - vram_lock -> reset_domain->sem: VRAM management paths may need to + * wait for ongoing reset to complete + * + * Note: mmap_lock ordering relative to GPU locks is already taught + * by dma-resv (drivers/dma-buf/dma-resv.c). + */ +int amdgpu_lockdep_init(void) +{ + struct amdgpu_reset_domain *reset_domain = NULL; + struct amdgpu_lockdep_dummy_locks *locks; + unsigned long flags; + + locks = kzalloc(sizeof(*locks), GFP_KERNEL); + if (!locks) + return -ENOMEM; + + /* + * Initialize dummy reset domain + */ + reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, + "lockdep_test"); + if (!reset_domain) { + kfree(locks); + return -ENOMEM; + } + /* Initialize dummy locks */ + mutex_init(&locks->userq_sch_mutex); + mutex_init(&locks->userq_mutex); + mutex_init(&locks->notifier_lock); + mutex_init(&locks->vram_lock); + mutex_init(&locks->reset_lock); + mutex_init(&locks->srbm_mutex); + mutex_init(&locks->grbm_idx_mutex); + spin_lock_init(&locks->mmio_idx_lock); + + /* + * Associate dummy locks with the same class keys used for real + * driver locks. This ensures lockdep connects the ordering learned + * here with the actual locks used at runtime. + */ + lockdep_set_class(&locks->userq_sch_mutex, &amdgpu_userq_sch_mutex_key); + lockdep_set_class(&locks->userq_mutex, &amdgpu_userq_mutex_key); + lockdep_set_class(&locks->notifier_lock, &amdgpu_notifier_lock_key); + lockdep_set_class(&locks->vram_lock, &amdgpu_vram_lock_key); + lockdep_set_class(&reset_domain->sem, &amdgpu_reset_sem_key); + lockdep_set_class(&locks->reset_lock, &amdgpu_reset_lock_key); + lockdep_set_class(&locks->srbm_mutex, &amdgpu_srbm_lock_key); + lockdep_set_class(&locks->grbm_idx_mutex, &amdgpu_grbm_lock_key); + lockdep_set_class(&locks->mmio_idx_lock, &amdgpu_mmio_lock_key); + /* + * Take locks in the correct order to train lockdep. + * This establishes the dependency chain. + */ + + /* Level 1: Global userq scheduler mutex (outermost) */ + mutex_lock(&locks->userq_sch_mutex); + + /* Level 2: Per-context userq mutex */ + mutex_lock(&locks->userq_mutex); + /* Level 3: MMU notifier lock */ + mutex_lock(&locks->notifier_lock); + /* Level 4: VRAM allocator lock */ + mutex_lock(&locks->vram_lock); + /* Level 5: Reset domain semaphore */ + down_read(&reset_domain->sem); + + /* Level 6: Reset control lock */ + mutex_lock(&locks->reset_lock); + /* + * Mark potential memory reclaim boundary. + * GPU operations might trigger memory allocation/reclaim. + */ + fs_reclaim_acquire(GFP_KERNEL); + + /* Level 7: SRBM register access */ + mutex_lock(&locks->srbm_mutex); + /* Level 8: GRBM index access */ + mutex_lock(&locks->grbm_idx_mutex); + + /* Level 9: MMIO index access (innermost lock, spinlock) */ + spin_lock_irqsave(&locks->mmio_idx_lock, flags); + /* + * All locks acquired in order. + * Lockdep has now learned the valid dependency chain. + */ + + /* Release in reverse order */ + spin_unlock_irqrestore(&locks->mmio_idx_lock, flags); + mutex_unlock(&locks->grbm_idx_mutex); + mutex_unlock(&locks->srbm_mutex); + fs_reclaim_release(GFP_KERNEL); + + mutex_unlock(&locks->reset_lock); + up_read(&reset_domain->sem); + + mutex_unlock(&locks->vram_lock); + mutex_unlock(&locks->notifier_lock); + mutex_unlock(&locks->userq_mutex); + mutex_unlock(&locks->userq_sch_mutex); + + /* Cleanup */ + amdgpu_reset_put_reset_domain(reset_domain); + + kfree(locks); + pr_info("AMDGPU: Lockdep annotations initialized (9 lock levels)\n"); + + return 0; +} + +#endif /* CONFIG_LOCKDEP */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.h new file mode 100644 index 000000000000..04adb58665bf --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright 2024 Advanced Micro Devices, Inc. + * + * Lockdep annotation interface for AMDGPU + */ + +#ifndef __AMDGPU_LOCKDEP_H__ +#define __AMDGPU_LOCKDEP_H__ + +#include <linux/lockdep.h> + +struct amdgpu_device; + +#ifdef CONFIG_LOCKDEP + +/** + * amdgpu_lockdep_init - Train lockdep on correct lock ordering + * + * Call once during module init to establish the lock dependency chain. + */ +int amdgpu_lockdep_init(void); + +/** + * amdgpu_lockdep_set_class - Associate lock class keys with real locks + * @adev: AMDGPU device + * + * Call during device init to associate lock classes with actual locks. + */ +void amdgpu_lockdep_set_class(struct amdgpu_device *adev); + +#else /* !CONFIG_LOCKDEP */ + +static inline int amdgpu_lockdep_init(void) { return 0; } +static inline void amdgpu_lockdep_set_class(struct amdgpu_device *adev) {} + +#endif /* CONFIG_LOCKDEP */ + +#endif /* __AMDGPU_LOCKDEP_H__ */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c index 823ba17e32af..cc6d1a4e4c3a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c @@ -99,6 +99,7 @@ int amdgpu_mca_mp0_ras_sw_init(struct amdgpu_device *adev) strcpy(ras->ras_block.ras_comm.name, "mca.mp0"); ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA; + ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MP0; ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->mca.mp0.ras_if = &ras->ras_block.ras_comm; @@ -123,6 +124,7 @@ int amdgpu_mca_mp1_ras_sw_init(struct amdgpu_device *adev) strcpy(ras->ras_block.ras_comm.name, "mca.mp1"); ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA; + ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MP1; ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->mca.mp1.ras_if = &ras->ras_block.ras_comm; @@ -147,6 +149,7 @@ int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev) strcpy(ras->ras_block.ras_comm.name, "mca.mpio"); ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA; + ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MPIO; ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; adev->mca.mpio.ras_if = &ras->ras_block.ras_comm; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c index 0d4c77c1b4b5..e3972673fd64 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c @@ -103,7 +103,7 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe, int amdgpu_mes_init(struct amdgpu_device *adev) { - int i, r, num_pipes; + int i, r, num_pipes, num_queues = 0; u32 total_vmid_mask, reserved_vmid_mask; int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; u32 gfx_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.me.num_pipe_per_me, @@ -159,7 +159,8 @@ int amdgpu_mes_init(struct amdgpu_device *adev) adev->mes.compute_hqd_mask[i] = compute_hqd_mask; } - num_pipes = adev->sdma.num_instances; + num_pipes = adev->sdma.num_inst_per_xcc ? + adev->sdma.num_inst_per_xcc : adev->sdma.num_instances; if (num_pipes > AMDGPU_MES_MAX_SDMA_PIPES) dev_warn(adev->dev, "more SDMA pipes than supported by MES! (%d vs %d)\n", num_pipes, AMDGPU_MES_MAX_SDMA_PIPES); @@ -216,8 +217,27 @@ int amdgpu_mes_init(struct amdgpu_device *adev) if (r) goto error_doorbell; + if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)) { + /* When queue/pipe reset is done in MES instead of in the + * driver, MES passes hung queues information to the driver in + * hung_queue_hqd_info. Calculate required space to store this + * information. + */ + for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) + num_queues += hweight32(adev->mes.gfx_hqd_mask[i]); + + for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) + num_queues += hweight32(adev->mes.compute_hqd_mask[i]); + + for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) + num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc; + + adev->mes.hung_queue_hqd_info_offset = num_queues; + adev->mes.hung_queue_db_array_size = num_queues * 2; + } + if (adev->mes.hung_queue_db_array_size) { - for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { + for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) { r = amdgpu_bo_create_kernel(adev, adev->mes.hung_queue_db_array_size * sizeof(u32), PAGE_SIZE, @@ -264,10 +284,10 @@ void amdgpu_mes_fini(struct amdgpu_device *adev) &adev->mes.event_log_cpu_addr); for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { - amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i], - &adev->mes.hung_queue_db_array_gpu_addr[i], - &adev->mes.hung_queue_db_array_cpu_addr[i]); - + if (adev->mes.hung_queue_db_array_gpu_obj[i]) + amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i], + &adev->mes.hung_queue_db_array_gpu_addr[i], + &adev->mes.hung_queue_db_array_cpu_addr[i]); if (adev->mes.sch_ctx_ptr[i]) amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); if (adev->mes.query_status_fence_ptr[i]) @@ -281,7 +301,7 @@ void amdgpu_mes_fini(struct amdgpu_device *adev) mutex_destroy(&adev->mes.mutex_hidden); } -int amdgpu_mes_suspend(struct amdgpu_device *adev) +int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id) { struct mes_suspend_gang_input input; int r; @@ -291,6 +311,10 @@ int amdgpu_mes_suspend(struct amdgpu_device *adev) memset(&input, 0x0, sizeof(struct mes_suspend_gang_input)); input.suspend_all_gangs = 1; + input.xcc_id = xcc_id; + if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) && + ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x71)) + input.suspend_all_sdma_gangs = 1; /* * Avoid taking any other locks under MES lock to avoid circular @@ -305,7 +329,7 @@ int amdgpu_mes_suspend(struct amdgpu_device *adev) return r; } -int amdgpu_mes_resume(struct amdgpu_device *adev) +int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id) { struct mes_resume_gang_input input; int r; @@ -315,6 +339,7 @@ int amdgpu_mes_resume(struct amdgpu_device *adev) memset(&input, 0x0, sizeof(struct mes_resume_gang_input)); input.resume_all_gangs = 1; + input.xcc_id = xcc_id; /* * Avoid taking any other locks under MES lock to avoid circular @@ -428,7 +453,7 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev, { struct mes_detect_and_reset_queue_input input; u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id]; - int r, i; + int hqd_info_offset = adev->mes.hung_queue_hqd_info_offset, r, i; if (!hung_db_num || !hung_db_array) return -EINVAL; @@ -443,26 +468,34 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev, adev->mes.hung_queue_db_array_size * sizeof(u32)); input.queue_type = queue_type; input.detect_only = detect_only; + input.xcc_id = xcc_id; r = adev->mes.funcs->detect_and_reset_hung_queues(&adev->mes, &input); - if (r) { - dev_err(adev->dev, "failed to detect and reset\n"); - } else { - *hung_db_num = 0; - for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) { - if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) { - hung_db_array[i] = db_array[i]; - *hung_db_num += 1; - } + + if (r && detect_only) { + dev_err(adev->dev, "Failed to detect hung queues\n"); + return r; + } + + *hung_db_num = 0; + /* MES passes hung queues' doorbell to driver */ + for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) { + /* Finding hung queues where db_array[i] is a valid doorbell */ + if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) { + hung_db_array[i] = db_array[i]; + *hung_db_num += 1; } + } - /* - * TODO: return HQD info for MES scheduled user compute queue reset cases - * stored in hung_db_array hqd info offset to full array size - */ + if (r && !(*hung_db_num)) { + dev_err(adev->dev, "Failed to detect and reset hung queues\n"); + return r; } + for (i = hqd_info_offset; i < hqd_info_offset + *hung_db_num; i++) + hung_db_array[i] = db_array[i]; + return r; } @@ -748,6 +781,18 @@ out: return r; } +void amdgpu_mes_validate_fw_version(struct amdgpu_device *adev) +{ + u32 fw_from_ucode = adev->mes.fw_version[AMDGPU_MES_SCHED_PIPE]; + u32 fw_from_reg = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; + + if (fw_from_ucode != fw_from_reg) + dev_info(adev->dev, + "MES firmware reports incorrect version in ucode binary (0x%x vs 0x%x)\n", + fw_from_ucode, fw_from_reg); +} + + bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev) { uint32_t mes_rev = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; @@ -758,6 +803,12 @@ bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev) amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 0, 0)); } +bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev) +{ + return (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) && + (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x73); +} + /* Fix me -- node_id is used to identify the correct MES instances in the future */ static int amdgpu_mes_set_enforce_isolation(struct amdgpu_device *adev, uint32_t node_id, bool enable) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h index f80e3aca9c78..1aae49f4df49 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h @@ -170,6 +170,19 @@ struct amdgpu_mes { uint64_t shared_cmd_buf_gpu_addr[AMDGPU_MAX_MES_INST_PIPES]; }; +struct amdgpu_mes_hung_queue_hqd_info { + union { + struct { + u32 queue_type: 3; // queue type + u32 pipe_index: 4; // pipe index + u32 queue_index: 8; // queue index + u32 reserved: 17; + }; + + u32 bit0_31; + }; +}; + struct amdgpu_mes_gang { int gang_id; int priority; @@ -285,15 +298,18 @@ struct mes_unmap_legacy_queue_input { struct mes_suspend_gang_input { uint32_t xcc_id; bool suspend_all_gangs; + bool suspend_all_sdma_gangs; uint64_t gang_context_addr; uint64_t suspend_fence_addr; uint32_t suspend_fence_value; + uint32_t doorbell_offset; }; struct mes_resume_gang_input { uint32_t xcc_id; bool resume_all_gangs; uint64_t gang_context_addr; + uint32_t doorbell_offset; }; struct mes_reset_queue_input { @@ -312,8 +328,9 @@ struct mes_reset_queue_input { }; struct mes_detect_and_reset_queue_input { - uint32_t queue_type; - bool detect_only; + u32 queue_type; + bool detect_only; + u32 xcc_id; }; struct mes_inv_tlbs_pasid_input { @@ -426,11 +443,12 @@ struct amdgpu_mes_funcs { (adev)->mes.kiq_hw_fini((adev), (xcc_id)) int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe); +void amdgpu_mes_validate_fw_version(struct amdgpu_device *adev); int amdgpu_mes_init(struct amdgpu_device *adev); void amdgpu_mes_fini(struct amdgpu_device *adev); -int amdgpu_mes_suspend(struct amdgpu_device *adev); -int amdgpu_mes_resume(struct amdgpu_device *adev); +int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id); +int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id); int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev, struct amdgpu_ring *ring, uint32_t xcc_id); @@ -534,6 +552,7 @@ static inline void amdgpu_mes_unlock(struct amdgpu_mes *mes) } bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev); +bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev); int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h index 51ab1a332615..8069fc41cc7f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h @@ -38,7 +38,6 @@ #include <drm/drm_probe_helper.h> #include <linux/i2c.h> #include <linux/i2c-algo-bit.h> -#include <linux/hrtimer.h> #include "amdgpu_irq.h" #include <drm/display/drm_dp_mst_helper.h> @@ -505,9 +504,6 @@ struct amdgpu_crtc { u32 line_time; u32 lb_vblank_lead_lines; struct drm_display_mode hw_mode; - /* for virtual dce */ - struct hrtimer vblank_timer; - enum amdgpu_interrupt_state vsync_timer_enabled; int otg_inst; struct drm_pending_vblank_event *event; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c index a974265837f0..e4c8e98725f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c @@ -84,3 +84,13 @@ late_fini: amdgpu_ras_block_late_fini(adev, ras_block); return r; } + + +void amdgpu_nbio_program_aspm(struct amdgpu_device *adev) +{ + if (!amdgpu_device_should_use_aspm(adev)) + return; + + if (adev->nbio.funcs->program_aspm) + adev->nbio.funcs->program_aspm(adev); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h index b528de6a01f6..a61f3a6e8ec7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h @@ -121,4 +121,6 @@ u64 amdgpu_nbio_get_pcie_replay_count(struct amdgpu_device *adev); bool amdgpu_nbio_is_replay_cnt_supported(struct amdgpu_device *adev); +void amdgpu_nbio_program_aspm(struct amdgpu_device *adev); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index b6aabac39b46..f98bfba59a2c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -717,13 +717,17 @@ int amdgpu_bo_create(struct amdgpu_device *adev, bo->tbo.resource->mem_type == TTM_PL_VRAM) { struct dma_fence *fence; - r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence); + r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev), + bo, bo->tbo.base.resv, &fence, + true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER); if (unlikely(r)) goto fail_unreserve; - dma_resv_add_fence(bo->tbo.base.resv, fence, - DMA_RESV_USAGE_KERNEL); - dma_fence_put(fence); + if (fence) { + dma_resv_add_fence(bo->tbo.base.resv, fence, + DMA_RESV_USAGE_KERNEL); + dma_fence_put(fence); + } } if (!bp->resv) amdgpu_bo_unreserve(bo); @@ -1325,9 +1329,9 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) if (r) goto out; - r = amdgpu_fill_buffer(amdgpu_ttm_next_clear_entity(adev), - abo, 0, &bo->base._resv, - &fence, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE); + r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev), + abo, &bo->base._resv, &fence, + false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE); if (WARN_ON(r)) goto out; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index f0e4d020f4c7..e0c0d7872e45 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -52,6 +52,17 @@ static int psp_load_smu_fw(struct psp_context *psp); static int psp_rap_terminate(struct psp_context *psp); static int psp_securedisplay_terminate(struct psp_context *psp); +static const char * const amdgpu_ptl_fmt_str[] = { + [AMDGPU_PTL_FMT_I8] = "I8", + [AMDGPU_PTL_FMT_F16] = "F16", + [AMDGPU_PTL_FMT_BF16] = "BF16", + [AMDGPU_PTL_FMT_F32] = "F32", + [AMDGPU_PTL_FMT_F64] = "F64", + [AMDGPU_PTL_FMT_F8] = "F8", + [AMDGPU_PTL_FMT_VECTOR] = "VECTOR", + [AMDGPU_PTL_FMT_INVALID] = "INVALID", +}; + static int psp_ring_init(struct psp_context *psp, enum psp_ring_type ring_type) { @@ -264,6 +275,7 @@ static int psp_early_init(struct amdgpu_ip_block *ip_block) psp->boot_time_tmr = false; break; case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 9): psp_v15_0_0_set_psp_funcs(psp); psp->boot_time_tmr = false; break; @@ -518,7 +530,7 @@ static int psp_sw_init(struct amdgpu_ip_block *ip_block) } ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG, - (amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf) ? + (amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf || adev->gmc.xgmi.connected_to_cpu) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT, &psp->fw_pri_bo, &psp->fw_pri_mc_addr, @@ -682,6 +694,8 @@ static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id) return "SPATIAL_PARTITION"; case GFX_CMD_ID_FB_NPS_MODE: return "NPS_MODE_CHANGE"; + case GFX_CMD_ID_PERF_HW: + return "PERF MONITORING HW"; default: return "UNKNOWN CMD"; } @@ -844,7 +858,11 @@ static int psp_load_toc(struct psp_context *psp, struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); /* Copy toc to psp firmware private buffer */ - psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes); + ret = psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes); @@ -1160,8 +1178,11 @@ static int psp_rl_load(struct amdgpu_device *adev) cmd = acquire_psp_cmd_buf(psp); - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes); + ret = psp_copy_fw(psp, psp->rl.start_addr, psp->rl.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr); @@ -1201,6 +1222,369 @@ int psp_memory_partition(struct psp_context *psp, int mode) return ret; } +static int psp_ptl_fmt_verify(struct psp_context *psp, enum amdgpu_ptl_fmt fmt, + uint32_t *ptl_fmt) +{ + struct amdgpu_device *adev = psp->adev; + + if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 4)) + return -EINVAL; + + switch (fmt) { + case AMDGPU_PTL_FMT_I8: + *ptl_fmt = GFX_FTYPE_I8; + break; + case AMDGPU_PTL_FMT_F16: + *ptl_fmt = GFX_FTYPE_F16; + break; + case AMDGPU_PTL_FMT_BF16: + *ptl_fmt = GFX_FTYPE_BF16; + break; + case AMDGPU_PTL_FMT_F32: + *ptl_fmt = GFX_FTYPE_F32; + break; + case AMDGPU_PTL_FMT_F64: + *ptl_fmt = GFX_FTYPE_F64; + break; + case AMDGPU_PTL_FMT_F8: + *ptl_fmt = GFX_FTYPE_F8; + break; + case AMDGPU_PTL_FMT_VECTOR: + *ptl_fmt = GFX_FTYPE_VECTOR; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int psp_ptl_invoke(struct psp_context *psp, u32 req_code, + uint32_t *ptl_state, uint32_t *fmt1, uint32_t *fmt2) +{ + struct psp_gfx_cmd_resp *cmd; + struct amdgpu_ptl *ptl = &psp->ptl; + int ret; + + cmd = acquire_psp_cmd_buf(psp); + + cmd->cmd_id = GFX_CMD_ID_PERF_HW; + cmd->cmd.cmd_req_perf_hw.req = req_code; + cmd->cmd.cmd_req_perf_hw.ptl_state = *ptl_state; + cmd->cmd.cmd_req_perf_hw.pref_format1 = *fmt1; + cmd->cmd.cmd_req_perf_hw.pref_format2 = *fmt2; + + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + if (ret) + goto out; + + /* + * Check response status explicitly to avoid + * updating cached PTL state with invalid data. + */ + if (cmd->resp.status) { + dev_err(psp->adev->dev, + "PTL command 0x%x failed, PSP response status: 0x%X fw resp=0x%X\n", + req_code, cmd->resp.status, + cmd->resp.uresp.perf_hw_info.resp); + ret = -EIO; + goto out; + } + + /* Parse response */ + switch (req_code) { + case PSP_PTL_PERF_MON_QUERY: + *ptl_state = cmd->resp.uresp.perf_hw_info.ptl_state; + *fmt1 = cmd->resp.uresp.perf_hw_info.pref_format1; + *fmt2 = cmd->resp.uresp.perf_hw_info.pref_format2; + dev_dbg(psp->adev->dev, "PTL query: state=%d, fmt1=%d, fmt2=%d\n", + *ptl_state, *fmt1, *fmt2); + break; + case PSP_PTL_PERF_MON_SET: + /* Update cached state only on success */ + ptl->enabled = *ptl_state; + ptl->fmt1 = *fmt1; + ptl->fmt2 = *fmt2; + dev_dbg(psp->adev->dev, "PTL set: state=%d, fmt1=%d, fmt2=%d\n", + *ptl_state, *fmt1, *fmt2); + break; + } + +out: + release_psp_cmd_buf(psp); + return ret; +} + +int amdgpu_ptl_perf_monitor_ctrl(struct amdgpu_device *adev, u32 req_code, + uint32_t *ptl_state, + enum amdgpu_ptl_fmt *fmt1, + enum amdgpu_ptl_fmt *fmt2) +{ + uint32_t ptl_fmt1, ptl_fmt2; + struct psp_context *psp; + struct amdgpu_ptl *ptl; + int ret; + + if (!adev || !ptl_state || !fmt1 || !fmt2) + return -EINVAL; + + if (amdgpu_sriov_vf(adev)) + return 0; + + psp = &adev->psp; + ptl = &psp->ptl; + + if (ptl->permanently_disabled && *ptl_state == 1) + return 0; + + if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 4) || + psp->sos.fw_version < 0x0036081a) + return -EOPNOTSUPP; + + /* Verify formats */ + if (psp_ptl_fmt_verify(psp, *fmt1, &ptl_fmt1) || + psp_ptl_fmt_verify(psp, *fmt2, &ptl_fmt2)) + return -EINVAL; + + /* + * Add check to skip if state and formats are identical to current ones + */ + if (req_code == PSP_PTL_PERF_MON_SET && + ptl->enabled == *ptl_state && + ptl->fmt1 == ptl_fmt1 && + ptl->fmt2 == ptl_fmt2) + return 0; + + /* If enabling PTL, check disable bitmap */ + if (req_code == PSP_PTL_PERF_MON_SET && *ptl_state == 1) { + if (!bitmap_empty(ptl->disable_bitmap, + AMDGPU_PTL_DISABLE_MAX)) { + dev_dbg(adev->dev, + "PTL enable blocked: SYSFS=%d, PROFILER=%d (ref=%d)\n", + test_bit(AMDGPU_PTL_DISABLE_SYSFS, + ptl->disable_bitmap), + test_bit(AMDGPU_PTL_DISABLE_PROFILER, + ptl->disable_bitmap), + atomic_read(&ptl->disable_ref)); + return 0; + } + } + + if (req_code == PSP_PTL_PERF_MON_SET) { + amdgpu_amdkfd_stop_sched_all(adev); + /* Wait for GFX engine idle before PTL state transition */ + ret = amdgpu_device_ip_wait_for_idle(adev, + AMD_IP_BLOCK_TYPE_GFX); + if (ret) { + amdgpu_amdkfd_start_sched_all(adev); + dev_err(adev->dev, "GFX not idle before PTL operation (%d)\n", ret); + return ret; + } + ret = psp_ptl_invoke(psp, req_code, ptl_state, &ptl_fmt1, &ptl_fmt2); + amdgpu_amdkfd_start_sched_all(adev); + } else { + ret = psp_ptl_invoke(psp, req_code, ptl_state, &ptl_fmt1, &ptl_fmt2); + } + + return ret; +} + +static enum amdgpu_ptl_fmt str_to_ptl_fmt(const char *str) +{ + int i; + + for (i = 0; i < AMDGPU_PTL_FMT_INVALID; ++i) { + if (!strcmp(str, amdgpu_ptl_fmt_str[i])) + return (enum amdgpu_ptl_fmt)i; + } + + return AMDGPU_PTL_FMT_INVALID; +} + +static ssize_t ptl_supported_formats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t len = 0; + + for (int i = 0; i < AMDGPU_PTL_FMT_INVALID; ++i) { + const char *fmt = amdgpu_ptl_fmt_str[i]; + + len += sysfs_emit_at(buf, len, "%s%s", + fmt ? fmt : "UNKNOWN", + (i < AMDGPU_PTL_FMT_INVALID - 1) ? "," : "\n"); + } + + return len; +} + +static ssize_t ptl_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + struct amdgpu_ptl *ptl = &adev->psp.ptl; + uint32_t ptl_state, fmt1, fmt2; + int ret; + bool enable; + bool bit_changed = false; + + mutex_lock(&ptl->mutex); + if (sysfs_streq(buf, "enabled") || sysfs_streq(buf, "1")) { + enable = true; + } else if (sysfs_streq(buf, "disabled") || sysfs_streq(buf, "0")) { + enable = false; + } else { + mutex_unlock(&ptl->mutex); + return -EINVAL; + } + + /* Block enable when permanently disabled */ + if (ptl->permanently_disabled) { + mutex_unlock(&ptl->mutex); + return -EPERM; + } + + fmt1 = ptl->fmt1; + fmt2 = ptl->fmt2; + ptl_state = enable ? 1 : 0; + + if (enable) + bit_changed = test_and_clear_bit(AMDGPU_PTL_DISABLE_SYSFS, + ptl->disable_bitmap); + + ret = amdgpu_ptl_perf_monitor_ctrl(adev, PSP_PTL_PERF_MON_SET, &ptl_state, &fmt1, &fmt2); + if (ret) { + dev_err(adev->dev, "Failed to set PTL err = %d\n", ret); + if (enable && bit_changed) + set_bit(AMDGPU_PTL_DISABLE_SYSFS, ptl->disable_bitmap); + mutex_unlock(&ptl->mutex); + return ret; + } + + if (!enable) + set_bit(AMDGPU_PTL_DISABLE_SYSFS, ptl->disable_bitmap); + + mutex_unlock(&ptl->mutex); + + return count; +} + +static ssize_t ptl_enable_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + struct amdgpu_ptl *ptl = &adev->psp.ptl; + + if (ptl->permanently_disabled) + return sysfs_emit(buf, "permanently disabled\n"); + + return sysfs_emit(buf, "%s\n", ptl->enabled ? "enabled" : "disabled"); +} + +static ssize_t ptl_format_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + char fmt1_str[8], fmt2_str[8]; + enum amdgpu_ptl_fmt fmt1_enum, fmt2_enum; + struct amdgpu_ptl *ptl = &adev->psp.ptl; + uint32_t ptl_state, fmt1, fmt2; + int ret; + + /* Only allow format update when PTL is enabled */ + if (!ptl->enabled) + return -EPERM; + + mutex_lock(&ptl->mutex); + /* Parse input, expecting "FMT1,FMT2" */ + if (sscanf(buf, "%7[^,],%7s", fmt1_str, fmt2_str) != 2) { + mutex_unlock(&ptl->mutex); + return -EINVAL; + } + + fmt1_enum = str_to_ptl_fmt(fmt1_str); + fmt2_enum = str_to_ptl_fmt(fmt2_str); + + if (fmt1_enum >= AMDGPU_PTL_FMT_INVALID || + fmt2_enum >= AMDGPU_PTL_FMT_INVALID || + fmt1_enum == fmt2_enum) { + mutex_unlock(&ptl->mutex); + return -EINVAL; + } + + ptl_state = ptl->enabled; + fmt1 = fmt1_enum; + fmt2 = fmt2_enum; + ret = amdgpu_ptl_perf_monitor_ctrl(adev, PSP_PTL_PERF_MON_SET, &ptl_state, &fmt1, &fmt2); + if (ret) { + dev_err(adev->dev, "Failed to update PTL err = %d\n", ret); + mutex_unlock(&ptl->mutex); + return ret; + } + mutex_unlock(&ptl->mutex); + + return count; +} + +static ssize_t ptl_format_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + struct psp_context *psp = &adev->psp; + + return sysfs_emit(buf, "%s,%s\n", + amdgpu_ptl_fmt_str[psp->ptl.fmt1], + amdgpu_ptl_fmt_str[psp->ptl.fmt2]); +} + +static umode_t amdgpu_ptl_is_visible(struct kobject *kobj, struct attribute *attr, int idx) +{ + struct device *dev = kobj_to_dev(kobj); + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + + /* Only show PTL sysfs files if PTL hardware is supported */ + if (!adev->psp.ptl.hw_supported) + return 0; + + return attr->mode; +} + +int amdgpu_ptl_sysfs_init(struct amdgpu_device *adev) +{ + struct amdgpu_ptl *ptl = &adev->psp.ptl; + int ret; + + if (!ptl->hw_supported) + return 0; + + if (ptl->ptl_sysfs_created) + return 0; + + ret = sysfs_create_group(&adev->dev->kobj, &amdgpu_ptl_attr_group); + if (!ret) + ptl->ptl_sysfs_created = true; + + return ret; +} + +void amdgpu_ptl_sysfs_fini(struct amdgpu_device *adev) +{ + struct amdgpu_ptl *ptl = &adev->psp.ptl; + + if (!ptl->hw_supported) + return; + + if (!ptl->ptl_sysfs_created) + return; + + sysfs_remove_group(&adev->dev->kobj, &amdgpu_ptl_attr_group); + ptl->ptl_sysfs_created = false; +} + int psp_spatial_partition(struct psp_context *psp, int mode) { struct psp_gfx_cmd_resp *cmd; @@ -1384,8 +1768,12 @@ int psp_ta_load(struct psp_context *psp, struct ta_context *context) cmd = acquire_psp_cmd_buf(psp); - psp_copy_fw(psp, context->bin_desc.start_addr, - context->bin_desc.size_bytes); + ret = psp_copy_fw(psp, context->bin_desc.start_addr, + context->bin_desc.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } if (amdgpu_virt_xgmi_migrate_enabled(psp->adev) && context->mem_context.shared_bo) @@ -3088,7 +3476,9 @@ static int psp_load_non_psp_fw(struct psp_context *psp) amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(15, 0, 0) || amdgpu_ip_version(adev, MP0_HWIP, 0) == - IP_VERSION(15, 0, 8)) && + IP_VERSION(15, 0, 8) || + amdgpu_ip_version(adev, MP0_HWIP, 0) == + IP_VERSION(15, 0, 9)) && (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3)) @@ -3527,7 +3917,12 @@ int psp_init_toc_microcode(struct psp_context *psp, const char *chip_name) const struct psp_firmware_header_v1_0 *toc_hdr; int err = 0; - err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED, + if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(15, 0, 8) && + adev->rev_id == 0) + err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED, + "amdgpu/%s_toc_1.bin", chip_name); + else + err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED, "amdgpu/%s_toc.bin", chip_name); if (err) goto out; @@ -4181,17 +4576,24 @@ fail: return count; } -void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size) +int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size) { int idx; if (!drm_dev_enter(adev_to_drm(psp->adev), &idx)) - return; + return -ENODEV; + + if (!bin_size || bin_size > PSP_1_MEG) { + dev_err(psp->adev->dev, "PSP firmware is invalid\n"); + drm_dev_exit(idx); + return -EINVAL; + } memset(psp->fw_pri_buf, 0, PSP_1_MEG); memcpy(psp->fw_pri_buf, start_addr, bin_size); drm_dev_exit(idx); + return 0; } /** @@ -4202,6 +4604,31 @@ void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size static DEVICE_ATTR(usbc_pd_fw, 0644, psp_usbc_pd_fw_sysfs_read, psp_usbc_pd_fw_sysfs_write); +/** + * DOC: PTL sysfs attributes + * These sysfs files under /sys/class/drm/cardX/device/ptl allow users to enable or disable + * the Peak Tops Limiter (PTL), configure preferred PTL data formats, and query supported + * formats for each GPU. + */ +static DEVICE_ATTR(ptl_enable, 0644, + ptl_enable_show, ptl_enable_store); +static DEVICE_ATTR(ptl_format, 0644, + ptl_format_show, ptl_format_store); +static DEVICE_ATTR(ptl_supported_formats, 0444, + ptl_supported_formats_show, NULL); + +static struct attribute *ptl_attrs[] = { + &dev_attr_ptl_enable.attr, + &dev_attr_ptl_format.attr, + &dev_attr_ptl_supported_formats.attr, + NULL, +}; + +const struct attribute_group amdgpu_ptl_attr_group = { + .name = "ptl", + .attrs = ptl_attrs, + .is_visible = amdgpu_ptl_is_visible, +}; int is_psp_fw_valid(struct psp_bin_desc bin) { @@ -4227,14 +4654,17 @@ static ssize_t amdgpu_psp_vbflash_write(struct file *filp, struct kobject *kobj, return -ENOMEM; } + mutex_lock(&adev->psp.mutex); + /* TODO Just allocate max for now and optimize to realloc later if needed */ if (!adev->psp.vbflash_tmp_buf) { adev->psp.vbflash_tmp_buf = kvmalloc(AMD_VBIOS_FILE_MAX_SIZE_B, GFP_KERNEL); - if (!adev->psp.vbflash_tmp_buf) + if (!adev->psp.vbflash_tmp_buf) { + mutex_unlock(&adev->psp.mutex); return -ENOMEM; + } } - mutex_lock(&adev->psp.mutex); memcpy(adev->psp.vbflash_tmp_buf + pos, buffer, count); adev->psp.vbflash_image_size += count; mutex_unlock(&adev->psp.mutex); @@ -4299,7 +4729,7 @@ rel_buf: */ static const struct bin_attribute psp_vbflash_bin_attr = { .attr = {.name = "psp_vbflash", .mode = 0660}, - .size = 0, + .size = AMD_VBIOS_FILE_MAX_SIZE_B, .write = amdgpu_psp_vbflash_write, .read = amdgpu_psp_vbflash_read, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 7e94ec11c57e..2d838b1b2b11 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -31,6 +31,8 @@ #include "ta_ras_if.h" #include "ta_rap_if.h" #include "ta_secureDisplay_if.h" +#include <linux/bitops.h> +#include "amdgpu_ptl.h" #define PSP_FENCE_BUFFER_SIZE 0x1000 #define PSP_CMD_BUFFER_SIZE 0x1000 @@ -358,6 +360,29 @@ struct spirom_bo { }; #endif +enum psp_ptl_cmd { + PSP_PTL_PERF_MON_QUERY = 0xA0000000, + PSP_PTL_PERF_MON_SET = 0xA0000001, +}; + +enum psp_ptl_format_type { + GFX_FTYPE_I8 = 0x00000000, + GFX_FTYPE_F16 = 0x00000001, + GFX_FTYPE_BF16 = 0x00000002, + GFX_FTYPE_F32 = 0x00000003, + GFX_FTYPE_F64 = 0x00000004, + GFX_FTYPE_F8 = 0x00000005, + GFX_FTYPE_VECTOR = 0x00000006, + GFX_FTYPE_INVALID = 0xFFFFFFFF, +}; + +struct psp_ptl_perf_req { + enum psp_ptl_cmd req; + uint32_t ptl_state; + uint32_t pref_format1; + uint32_t pref_format2; +}; + struct psp_context { struct amdgpu_device *adev; struct psp_ring km_ring; @@ -448,6 +473,7 @@ struct psp_context { #if defined(CONFIG_DEBUG_FS) struct spirom_bo *spirom_dump_trip; #endif + struct amdgpu_ptl ptl; }; struct amdgpu_psp_funcs { @@ -612,7 +638,7 @@ int psp_get_fw_attestation_records_addr(struct psp_context *psp, int psp_update_fw_reservation(struct psp_context *psp); int psp_load_fw_list(struct psp_context *psp, struct amdgpu_firmware_info **ucode_list, int ucode_count); -void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size); +int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size); int psp_spatial_partition(struct psp_context *psp, int mode); int psp_memory_partition(struct psp_context *psp, int mode); @@ -631,5 +657,4 @@ void amdgpu_psp_debugfs_init(struct amdgpu_device *adev); int amdgpu_psp_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *type); - #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index fc9f3adf9912..764cd4950408 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -95,6 +95,9 @@ struct amdgpu_ras_block_list { struct list_head node; struct amdgpu_ras_block_object *ras_obj; + + /* set by ras_late_init, cleared by ras_suspend/ras_fini */ + bool active; }; const char *get_ras_block_str(struct ras_common_if *ras_block) @@ -3094,6 +3097,25 @@ static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, return -EINVAL; } +static bool __check_record_in_range(struct amdgpu_device *adev, + struct eeprom_table_record *bps, int count) +{ + int i; + + for (i = 0; i < count; i++) { + if (bps[i].retired_page >= + (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT)) { + dev_warn(adev->dev, + "Recorded address out of range: 0x%llx, 0x%llx, 0x%x, 0x%x\n", + bps[i].address, bps[i].retired_page, + bps[i].mem_channel, bps[i].mcumc_id); + return false; + } + } + + return true; +} + static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct eeprom_table_record *bps, int count) { @@ -3101,6 +3123,9 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct amdgpu_ras *con = amdgpu_ras_get_context(adev); struct ras_err_handler_data *data = con->eh_data; + if (!__check_record_in_range(adev, bps, count)) + return 0; + for (j = 0; j < count; j++) { if (!data->space_left && amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { @@ -4615,10 +4640,23 @@ void amdgpu_ras_resume(struct amdgpu_device *adev) void amdgpu_ras_suspend(struct amdgpu_device *adev) { struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct amdgpu_ras_block_list *node; + struct amdgpu_ras_block_object *obj; if (!adev->ras_enabled || !con) return; + /* run per-block ras_suspend before tearing down the RAS context */ + list_for_each_entry(node, &adev->ras_list, node) { + if (!node->active) + continue; + + obj = node->ras_obj; + if (obj && obj->ras_suspend) + obj->ras_suspend(adev, &obj->ras_comm); + node->active = false; + } + amdgpu_ras_disable_all_features(adev, 0); /* Make sure all ras objects are disabled. */ if (AMDGPU_RAS_GET_FEATURES(con->features)) @@ -4672,8 +4710,15 @@ int amdgpu_ras_late_init(struct amdgpu_device *adev) obj->ras_comm.name, r); return r; } - } else - amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); + } else { + r = amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); + if (r) { + dev_err(adev->dev, "%s failed to execute ras_block_late_init_default! ret:%d\n", + obj->ras_comm.name, r); + return r; + } + } + node->active = true; } amdgpu_ras_check_bad_page_status(adev); @@ -4712,11 +4757,12 @@ int amdgpu_ras_fini(struct amdgpu_device *adev) list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { if (ras_node->ras_obj) { obj = ras_node->ras_obj; - if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && - obj->ras_fini) + /* fall back to default cleanup if ras_suspend already ran */ + if (ras_node->active && obj->ras_fini) obj->ras_fini(adev, &obj->ras_comm); else amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); + ras_node->active = false; } /* Clear ras blocks from ras_list and free ras block list node */ @@ -5642,6 +5688,11 @@ int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; int ret = 0; + if (pfn >= (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT)) { + dev_warn(adev->dev, "Ignoring out-of-range bad page 0x%llx", start); + return 0; + } + if (amdgpu_ras_check_critical_address(adev, start)) return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h index ff44190d7d98..a86ab65aa2f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h @@ -762,6 +762,7 @@ struct amdgpu_ras_block_object { int (*ras_block_match)(struct amdgpu_ras_block_object *block_obj, enum amdgpu_ras_block block, uint32_t sub_block_index); int (*ras_late_init)(struct amdgpu_device *adev, struct ras_common_if *ras_block); + void (*ras_suspend)(struct amdgpu_device *adev, struct ras_common_if *ras_block); void (*ras_fini)(struct amdgpu_device *adev, struct ras_common_if *ras_block); ras_ih_cb ras_cb; const struct amdgpu_ras_block_hw_ops *hw_ops; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 0c57fe259894..b265b4d9053f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -145,12 +145,15 @@ #define RAS_RI_TO_AI(_C, _I) (((_I) + (_C)->ras_fri) % \ (_C)->ras_max_record_count) -#define RAS_NUM_RECS(_tbl_hdr) (((_tbl_hdr)->tbl_size - \ - RAS_TABLE_HEADER_SIZE) / RAS_TABLE_RECORD_SIZE) +#define RAS_NUM_RECS(_tbl_hdr) \ + (((_tbl_hdr)->tbl_size < RAS_TABLE_HEADER_SIZE) ? 0u : \ + (((_tbl_hdr)->tbl_size - RAS_TABLE_HEADER_SIZE) / RAS_TABLE_RECORD_SIZE)) -#define RAS_NUM_RECS_V2_1(_tbl_hdr) (((_tbl_hdr)->tbl_size - \ - RAS_TABLE_HEADER_SIZE - \ - RAS_TABLE_V2_1_INFO_SIZE) / RAS_TABLE_RECORD_SIZE) +#define RAS_NUM_RECS_V2_1(_tbl_hdr) \ + (((_tbl_hdr)->tbl_size < RAS_TABLE_HEADER_SIZE + \ + RAS_TABLE_V2_1_INFO_SIZE) ? 0u : \ + (((_tbl_hdr)->tbl_size - RAS_TABLE_HEADER_SIZE - \ + RAS_TABLE_V2_1_INFO_SIZE) / RAS_TABLE_RECORD_SIZE)) #define to_amdgpu_device(x) ((container_of(x, struct amdgpu_ras, eeprom_control))->adev) @@ -1051,6 +1054,7 @@ int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control, uint64_t ts, end_idx; int i, ret; u64 mca, ipid; + u32 cu, mem_channel, mcumc_id; if (!amdgpu_ras_smu_eeprom_supported(adev)) return 0; @@ -1079,9 +1083,10 @@ int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control, record[i - rec_idx].err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE; adev->umc.ras->mca_ipid_parse(adev, ipid, - (uint32_t *)&(record[i - rec_idx].cu), - (uint32_t *)&(record[i - rec_idx].mem_channel), - (uint32_t *)&(record[i - rec_idx].mcumc_id), NULL); + &cu, &mem_channel, &mcumc_id, NULL); + record[i - rec_idx].cu = (u8)cu; + record[i - rec_idx].mem_channel = (u8)mem_channel; + record[i - rec_idx].mcumc_id = (u8)mcumc_id; } return 0; @@ -1608,11 +1613,24 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) switch (hdr->version) { case RAS_TABLE_VER_V2_1: case RAS_TABLE_VER_V3: + if (hdr->tbl_size < RAS_TABLE_HEADER_SIZE + RAS_TABLE_V2_1_INFO_SIZE) { + dev_err(adev->dev, + "RAS header invalid, tbl_size %u smaller than minimum %u, resetting table\n", + hdr->tbl_size, + RAS_TABLE_HEADER_SIZE + RAS_TABLE_V2_1_INFO_SIZE); + return amdgpu_ras_eeprom_reset_table(control); + } control->ras_num_recs = RAS_NUM_RECS_V2_1(hdr); control->ras_record_offset = RAS_RECORD_START_V2_1; control->ras_max_record_count = RAS_MAX_RECORD_COUNT_V2_1; break; case RAS_TABLE_VER_V1: + if (hdr->tbl_size < RAS_TABLE_HEADER_SIZE) { + dev_err(adev->dev, + "RAS header invalid, tbl_size %u smaller than minimum %u, resetting table\n", + hdr->tbl_size, RAS_TABLE_HEADER_SIZE); + return amdgpu_ras_eeprom_reset_table(control); + } control->ras_num_recs = RAS_NUM_RECS(hdr); control->ras_record_offset = RAS_RECORD_START; control->ras_max_record_count = RAS_MAX_RECORD_COUNT; @@ -1632,6 +1650,14 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) } control->ras_fri = RAS_OFFSET_TO_INDEX(control, hdr->first_rec_offset); + if (hdr->first_rec_offset < control->ras_record_offset || + control->ras_fri >= control->ras_max_record_count) { + dev_err(adev->dev, + "RAS header invalid, ras_fri: %u, first_rec_offset:0x%x", + control->ras_fri, hdr->first_rec_offset); + return -EINVAL; + } + control->ras_num_mca_recs = 0; control->ras_num_pa_recs = 0; return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c index 540040c76058..7468855c16a2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c @@ -406,7 +406,10 @@ uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) if (offset < adev->rmmio_size) return (readb(adev->rmmio + offset)); - BUG(); + + dev_err(adev->dev, "invalid MMIO read offset 0x%x (rmmio size 0x%x)\n", + offset, (unsigned int)adev->rmmio_size); + return 0; } /** @@ -469,10 +472,13 @@ void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) if (amdgpu_device_skip_hw_access(adev)) return; - if (offset < adev->rmmio_size) + if (offset < adev->rmmio_size) { writeb(value, adev->rmmio + offset); - else - BUG(); + } else { + dev_err(adev->dev, "invalid MMIO write offset 0x%x (rmmio size 0x%x)\n", + offset, (unsigned int)adev->rmmio_size); + return; + } } /** @@ -956,3 +962,21 @@ uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst, } return ret; } + + +uint32_t amdgpu_read_indexed_register(struct amdgpu_device *adev, + u32 se_num, u32 sh_num, u32 reg_offset) +{ + uint32_t val; + + mutex_lock(&adev->grbm_idx_mutex); + if (se_num != 0xffffffff || sh_num != 0xffffffff) + amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0); + + val = RREG32(reg_offset); + + if (se_num != 0xffffffff || sh_num != 0xffffffff) + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); + mutex_unlock(&adev->grbm_idx_mutex); + return val; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h index 4d88e5cd19fc..a1011af6b52b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h @@ -160,4 +160,7 @@ uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst, uint32_t reg_addr, char reg_name[], uint32_t expected_value, uint32_t mask); +uint32_t amdgpu_read_indexed_register(struct amdgpu_device *adev, + u32 se_num, u32 sh_num, u32 reg_offset); + #endif /* __AMDGPU_REG_ACCESS_H__ */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c index 1b982b803e6f..428c3cbc4a40 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c @@ -90,6 +90,7 @@ static int amdgpu_reset_xgmi_reset_on_init_restore_hwctxt( kgd2kfd_init_zone_device(tmp_adev); amdgpu_amdkfd_device_init(tmp_adev); amdgpu_amdkfd_drm_client_create(tmp_adev); + amdgpu_ptl_sysfs_init(tmp_adev); } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h index 07b4d37f1db6..c9f23a8e8db8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h @@ -46,6 +46,47 @@ enum AMDGPU_RESET_SRCS { AMDGPU_RESET_SRC_USERQ, }; +/** + * enum amd_reset_method - Methods for resetting AMD GPU devices + * + * @AMD_RESET_METHOD_NONE: The device will not be reset. + * @AMD_RESET_METHOD_LEGACY: Method reserved for SI, CIK and VI ASICs. + * @AMD_RESET_METHOD_MODE0: Reset the entire ASIC. Not currently available for + * the any device. + * @AMD_RESET_METHOD_MODE1: Resets all IP blocks on the ASIC (SDMA, GFX, VCN, + * etc.) individually. Suitable only for some discrete GPU, + * not available for all ASICs. + * @AMD_RESET_METHOD_MODE2: Resets a lesser level of IPs compared to MODE1. + * Which IPs are reset depends on the ASIC. Notably doesn't + * reset IPs shared with the CPU on APUs or the memory + * controllers (so VRAM is not lost). Not available on all + * ASICs. + * @AMD_RESET_METHOD_LINK: Triggers SW-UP link reset on other GPUs + * @AMD_RESET_METHOD_BACO: BACO (Bus Alive, Chip Off) method powers off and on + * the card but without powering off the PCI bus. Suitable + * only for discrete GPUs. + * @AMD_RESET_METHOD_PCI: Does a full bus reset using core Linux subsystem + * PCI reset and does a secondary bus reset or FLR, + * depending on what the underlying hardware supports. + * @AMD_RESET_METHOD_ON_INIT: Does a device reset during the driver init + * sequence. + * + * Methods available for AMD GPU driver for resetting the device. Not all + * methods are suitable for every device. User can override the method using + * module parameter `reset_method`. + */ +enum amd_reset_method { + AMD_RESET_METHOD_NONE = -1, + AMD_RESET_METHOD_LEGACY = 0, + AMD_RESET_METHOD_MODE0, + AMD_RESET_METHOD_MODE1, + AMD_RESET_METHOD_MODE2, + AMD_RESET_METHOD_LINK, + AMD_RESET_METHOD_BACO, + AMD_RESET_METHOD_PCI, + AMD_RESET_METHOD_ON_INIT, +}; + struct amdgpu_reset_context { enum amd_reset_method method; struct amdgpu_device *reset_req_dev; @@ -56,6 +97,20 @@ struct amdgpu_reset_context { enum AMDGPU_RESET_SRCS src; }; +struct amdgpu_reset_control { + void *handle; + struct work_struct reset_work; + struct mutex reset_lock; + struct amdgpu_reset_handler *( + *reset_handlers)[AMDGPU_RESET_MAX_HANDLERS]; + atomic_t in_reset; + enum amd_reset_method active_reset; + struct amdgpu_reset_handler *(*get_reset_handler)( + struct amdgpu_reset_control *reset_ctl, + struct amdgpu_reset_context *context); + void (*async_reset)(struct work_struct *work); +}; + struct amdgpu_reset_handler { enum amd_reset_method reset_method; int (*prepare_env)(struct amdgpu_reset_control *reset_ctl, @@ -72,20 +127,6 @@ struct amdgpu_reset_handler { int (*do_reset)(struct amdgpu_device *adev); }; -struct amdgpu_reset_control { - void *handle; - struct work_struct reset_work; - struct mutex reset_lock; - struct amdgpu_reset_handler *( - *reset_handlers)[AMDGPU_RESET_MAX_HANDLERS]; - atomic_t in_reset; - enum amd_reset_method active_reset; - struct amdgpu_reset_handler *(*get_reset_handler)( - struct amdgpu_reset_control *reset_ctl, - struct amdgpu_reset_context *context); - void (*async_reset)(struct work_struct *work); -}; - enum amdgpu_reset_domain_type { SINGLE_DEVICE, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index d6bee5c30073..b97fa35bac23 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -484,7 +484,7 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, static ssize_t amdgpu_ras_cper_debugfs_read(struct file *f, char __user *buf, size_t size, loff_t *offset) { - const uint8_t ring_header_size = 12; + const u8 ring_header_size = 12; struct amdgpu_ring *ring = file_inode(f)->i_private; struct ras_cmd_cper_snapshot_req *snapshot_req __free(kfree) = kzalloc_obj(struct ras_cmd_cper_snapshot_req); @@ -494,49 +494,103 @@ static ssize_t amdgpu_ras_cper_debugfs_read(struct file *f, char __user *buf, kzalloc_obj(struct ras_cmd_cper_record_req); struct ras_cmd_cper_record_rsp *record_rsp __free(kfree) = kzalloc_obj(struct ras_cmd_cper_record_rsp); - uint8_t *ring_header __free(kfree) = + u32 *ring_header __free(kfree) = kzalloc(ring_header_size, GFP_KERNEL); - uint32_t total_cper_num; - uint64_t start_cper_id; + char __user *data_buf = buf; + size_t data_size = size; + u32 total_cper_num; + u64 start_cper_id; + u64 cper_offset; + size_t chunk_size; + size_t total_data_size = 0; + bool read_header; int r; if (!snapshot_req || !snapshot_rsp || !record_req || !record_rsp || !ring_header) return -ENOMEM; - if (!(*offset)) { + read_header = !(*offset); + cper_offset = read_header ? 0 : *offset - 1; + + if (read_header) { /* Need at least 12 bytes for the header on the first read */ if (size < ring_header_size) return -EINVAL; - - if (copy_to_user(buf, ring_header, ring_header_size)) - return -EFAULT; - buf += ring_header_size; - size -= ring_header_size; + data_buf += ring_header_size; + data_size -= ring_header_size; } r = amdgpu_ras_mgr_handle_ras_cmd(ring->adev, RAS_CMD__GET_CPER_SNAPSHOT, snapshot_req, sizeof(struct ras_cmd_cper_snapshot_req), snapshot_rsp, sizeof(struct ras_cmd_cper_snapshot_rsp)); - if (r || !snapshot_rsp->total_cper_num) + if (r) return r; + if (!snapshot_rsp->total_cper_num) { + if (!read_header) + return 0; + + if (copy_to_user(buf, ring_header, ring_header_size)) + return -EFAULT; + + *offset = 1; + return ring_header_size; + } + start_cper_id = snapshot_rsp->start_cper_id; total_cper_num = snapshot_rsp->total_cper_num; + if (read_header && !data_size) { + if (copy_to_user(buf, ring_header, ring_header_size)) + return -EFAULT; - record_req->buf_ptr = (uint64_t)(uintptr_t)buf; - record_req->buf_size = size; - record_req->cper_start_id = start_cper_id + *offset; - record_req->cper_num = total_cper_num; - r = amdgpu_ras_mgr_handle_ras_cmd(ring->adev, RAS_CMD__GET_CPER_RECORD, - record_req, sizeof(struct ras_cmd_cper_record_req), - record_rsp, sizeof(struct ras_cmd_cper_record_rsp)); - if (r) - return r; + *offset = cper_offset + 1; + return ring_header_size; + } + + if (!data_size) + return 0; + + while (data_size && cper_offset < total_cper_num) { + memset(record_req, 0, sizeof(*record_req)); + memset(record_rsp, 0, sizeof(*record_rsp)); + chunk_size = min_t(size_t, data_size, RAS_CMD_MAX_CPER_BUF_SZ); + + record_req->buf_ptr = (u64)(uintptr_t)data_buf; + record_req->buf_size = chunk_size; + record_req->cper_start_id = start_cper_id + cper_offset; + record_req->cper_num = total_cper_num - cper_offset; + r = amdgpu_ras_mgr_handle_ras_cmd(ring->adev, + RAS_CMD__GET_CPER_RECORD, + record_req, + sizeof(struct ras_cmd_cper_record_req), + record_rsp, + sizeof(struct ras_cmd_cper_record_rsp)); + if (r) + return r; + + if (!record_rsp->real_data_size || !record_rsp->real_cper_num) + break; + if (record_rsp->real_data_size > data_size) + return -EIO; + + data_buf += record_rsp->real_data_size; + data_size -= record_rsp->real_data_size; + total_data_size += record_rsp->real_data_size; + cper_offset += record_rsp->real_cper_num; + } + + if (read_header) { + ring_header[1] = total_data_size >> 2; + ring_header[2] = ring_header[1]; + + if (copy_to_user(buf, ring_header, ring_header_size)) + return -EFAULT; + } - r = *offset ? record_rsp->real_data_size : record_rsp->real_data_size + ring_header_size; - (*offset) += record_rsp->real_cper_num; + r = read_header ? total_data_size + ring_header_size : total_data_size; + *offset = cper_offset + 1; return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index 715c9e43e13a..8f28b3bd7010 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -159,7 +159,8 @@ struct amdgpu_fence { extern const struct drm_sched_backend_ops amdgpu_sched_ops; void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error); -void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring); +void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring, + struct dma_fence *timedout_fence); void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring, struct amdgpu_fence *guilty_fence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c index 321310ba2c08..fcd81242059e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c @@ -600,10 +600,10 @@ exit: * to be submitted to the queues after the reset is complete. */ if (!ret) { - amdgpu_fence_driver_force_completion(gfx_ring); + amdgpu_fence_driver_force_completion(gfx_ring, NULL); drm_sched_wqueue_start(&gfx_ring->sched); if (adev->sdma.has_page_queue) { - amdgpu_fence_driver_force_completion(page_ring); + amdgpu_fence_driver_force_completion(page_ring, NULL); drm_sched_wqueue_start(&page_ring->sched); } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c index 3739be1b71e0..ab73e26c97e0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c @@ -98,15 +98,18 @@ static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __u uint32_t phy_id; uint32_t op; char str[64]; - int ret; + long len; + ssize_t write_ret = size; + int ret, nargs; - if (*pos || size > sizeof(str) - 1) + if (*pos) return -EINVAL; - memset(str, 0, sizeof(str)); - ret = copy_from_user(str, buf, size); - if (ret) + len = strncpy_from_user(str, buf, sizeof(str)); + if (len < 0) return -EFAULT; + if (len == 0 || len >= sizeof(str)) + return -EINVAL; ret = pm_runtime_get_sync(dev->dev); if (ret < 0) { @@ -114,10 +117,14 @@ static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __u return ret; } - if (size < 3) - sscanf(str, "%u ", &op); + if (len < 3) + nargs = sscanf(str, "%u", &op); else - sscanf(str, "%u %u", &op, &phy_id); + nargs = sscanf(str, "%u %u", &op, &phy_id); + if (nargs < 1) { + write_ret = -EINVAL; + goto out; + } switch (op) { case 1: @@ -135,9 +142,10 @@ static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __u mutex_unlock(&psp->securedisplay_context.mutex); break; case 2: - if (size < 3 || phy_id >= TA_SECUREDISPLAY_MAX_PHY) { + if (nargs < 2 || phy_id >= TA_SECUREDISPLAY_MAX_PHY) { dev_err(adev->dev, "Invalid input: %s\n", str); - return -EINVAL; + write_ret = -EINVAL; + break; } mutex_lock(&psp->securedisplay_context.mutex); psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, @@ -157,11 +165,14 @@ static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __u break; default: dev_err(adev->dev, "Invalid input: %s\n", str); + write_ret = -EINVAL; + break; } +out: pm_runtime_put_autosuspend(dev->dev); - return size; + return write_ret; } static const struct file_operations amdgpu_securedisplay_debugfs_ops = { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index d13e64a69e25..85724ec6aaf8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h @@ -145,6 +145,7 @@ TRACE_EVENT(amdgpu_cs, struct amdgpu_ib *ib), TP_ARGS(p, job, ib), TP_STRUCT__entry( + __field(struct drm_gpu_scheduler *, sched) __field(struct amdgpu_bo_list *, bo_list) __field(u32, ring) __field(u32, dw) @@ -152,11 +153,14 @@ TRACE_EVENT(amdgpu_cs, ), TP_fast_assign( + __entry->sched = container_of(job->base.entity->rq, + typeof(*__entry->sched), + rq); __entry->bo_list = p->bo_list; - __entry->ring = to_amdgpu_ring(job->base.entity->rq->sched)->idx; + __entry->ring = to_amdgpu_ring(__entry->sched)->idx; __entry->dw = ib->length_dw; __entry->fences = amdgpu_fence_count_emitted( - to_amdgpu_ring(job->base.entity->rq->sched)); + to_amdgpu_ring(__entry->sched)); ), TP_printk("bo_list=%p, ring=%u, dw=%u, fences=%u", __entry->bo_list, __entry->ring, __entry->dw, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 3d2e00efc741..025625e7e800 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -171,7 +171,7 @@ amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entit { struct amdgpu_ring *ring; - ring = adev->mman.buffer_funcs_ring; + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); amdgpu_ring_pad_ib(ring, &job->ibs[0]); WARN_ON(job->ibs[0].length_dw > num_dw); @@ -208,9 +208,10 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity, void *cpu_addr; uint64_t flags; int r; + const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT); BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < - AMDGPU_GTT_MAX_TRANSFER_SIZE * 8); + GTT_MAX_PAGES * AMDGPU_GPU_PAGES_IN_CPU_PAGE * 8); if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT)) return -EINVAL; @@ -230,7 +231,7 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity, offset = mm_cur->start & ~PAGE_MASK; num_pages = PFN_UP(*size + offset); - num_pages = min_t(uint32_t, num_pages, AMDGPU_GTT_MAX_TRANSFER_SIZE); + num_pages = min_t(uint32_t, num_pages, GTT_MAX_PAGES); *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset); @@ -420,8 +421,8 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo, if (old_mem->mem_type == TTM_PL_VRAM && (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) { struct dma_fence *wipe_fence = NULL; - r = amdgpu_fill_buffer(entity, abo, 0, NULL, &wipe_fence, - AMDGPU_KERNEL_JOB_ID_MOVE_BLIT); + r = amdgpu_ttm_clear_buffer(entity, abo, NULL, &wipe_fence, + false, AMDGPU_KERNEL_JOB_ID_MOVE_BLIT); if (r) { goto error; } else if (wipe_fence) { @@ -515,6 +516,15 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, if (new_mem->mem_type == TTM_PL_TT || new_mem->mem_type == AMDGPU_PL_PREEMPT) { + if (old_mem && (old_mem->mem_type == TTM_PL_TT || + old_mem->mem_type == AMDGPU_PL_PREEMPT)) { + r = ttm_bo_wait_ctx(bo, ctx); + if (r) + return r; + + amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); + } + r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); if (r) return r; @@ -549,6 +559,15 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, ttm_bo_assign_mem(bo, new_mem); return 0; } + if ((old_mem->mem_type == TTM_PL_TT || + old_mem->mem_type == AMDGPU_PL_PREEMPT) && + (new_mem->mem_type == TTM_PL_TT || + new_mem->mem_type == AMDGPU_PL_PREEMPT)) { + amdgpu_bo_move_notify(bo, evict, new_mem); + ttm_resource_free(bo, &bo->resource); + ttm_bo_assign_mem(bo, new_mem); + return 0; + } if (old_mem->mem_type == AMDGPU_PL_GDS || old_mem->mem_type == AMDGPU_PL_GWS || @@ -1563,7 +1582,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo, if (!adev->mman.sdma_access_ptr) return -EACCES; - if (!drm_dev_enter(adev_to_drm(adev), &idx)) + if (!adev->mman.buffer_funcs_enabled || !drm_dev_enter(adev_to_drm(adev), &idx)) return -ENODEV; if (write) @@ -1715,10 +1734,7 @@ static void amdgpu_ttm_init_fw_resv_region(struct amdgpu_device *adev) reserve_size = max(reserve_size, (uint32_t)280 << 20); else if (!adev->bios && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) { - if (hweight32(adev->aid_mask) == 1) - reserve_size = max(reserve_size, (uint32_t)128 << 20); - else - reserve_size = max(reserve_size, (uint32_t)144 << 20); + reserve_size = max(reserve_size, (uint32_t)150 << 20); } else if (!reserve_size) reserve_size = DISCOVERY_TMR_OFFSET; @@ -2018,6 +2034,7 @@ static int amdgpu_ttm_buffer_entity_init(struct amdgpu_gtt_mgr *mgr, u32 num_gart_windows) { int i, r, num_pages; + const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT); r = drm_sched_entity_init(&entity->base, prio, scheds, num_schedulers, NULL); if (r) @@ -2030,7 +2047,7 @@ static int amdgpu_ttm_buffer_entity_init(struct amdgpu_gtt_mgr *mgr, if (num_gart_windows == 0) return 0; - num_pages = num_gart_windows * AMDGPU_GTT_MAX_TRANSFER_SIZE; + num_pages = num_gart_windows * GTT_MAX_PAGES; r = amdgpu_gtt_mgr_alloc_entries(mgr, &entity->gart_node, num_pages, DRM_MM_INSERT_BEST); if (r) { @@ -2041,7 +2058,7 @@ static int amdgpu_ttm_buffer_entity_init(struct amdgpu_gtt_mgr *mgr, for (i = 0; i < num_gart_windows; i++) { entity->gart_window_offs[i] = amdgpu_gtt_node_to_byte_offset(&entity->gart_node) + - i * AMDGPU_GTT_MAX_TRANSFER_SIZE * PAGE_SIZE; + i * GTT_MAX_PAGES * PAGE_SIZE; } return 0; @@ -2101,20 +2118,25 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) } /* Change the size here instead of the init above so only lpfn is affected */ - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_ttm_disable_buffer_funcs(adev); #ifdef CONFIG_64BIT -#ifdef CONFIG_X86 - if (adev->gmc.xgmi.connected_to_cpu) - adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, - adev->gmc.visible_vram_size); - - else if (adev->gmc.is_app_apu) + if (adev->gmc.xgmi.connected_to_cpu) { + void *kaddr = devm_memremap(adev->dev, adev->gmc.aper_base, + adev->gmc.visible_vram_size, + MEMREMAP_WB); + if (IS_ERR(kaddr)) + return PTR_ERR(kaddr); + adev->mman.aper_base_kaddr = (__force void __iomem *)kaddr; + } else if (adev->gmc.is_app_apu) { DRM_DEBUG_DRIVER( "No need to ioremap when real vram size is 0\n"); - else -#endif - adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, - adev->gmc.visible_vram_size); + } else { + adev->mman.aper_base_kaddr = devm_ioremap_wc(adev->dev, + adev->gmc.aper_base, + adev->gmc.visible_vram_size); + if (!adev->mman.aper_base_kaddr) + return -ENOMEM; + } #endif amdgpu_ttm_init_vram_resv_regions(adev); @@ -2231,8 +2253,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) */ void amdgpu_ttm_fini(struct amdgpu_device *adev) { - int idx; - if (!adev->mman.initialized) return; @@ -2255,14 +2275,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_FW_VRAM_USAGE); amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_DRV_VRAM_USAGE); - if (drm_dev_enter(adev_to_drm(adev), &idx)) { - - if (adev->mman.aper_base_kaddr) - iounmap(adev->mman.aper_base_kaddr); - adev->mman.aper_base_kaddr = NULL; - - drm_dev_exit(idx); - } + adev->mman.aper_base_kaddr = NULL; if (!adev->gmc.is_app_apu) amdgpu_vram_mgr_fini(adev); @@ -2281,115 +2294,92 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) } /** - * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions + * amdgpu_ttm_enable_buffer_funcs - enable use of buffer functions * * @adev: amdgpu_device pointer - * @enable: true when we can use buffer functions. * - * Enable/disable use of buffer functions during suspend/resume. This should + * Enable use of buffer functions during suspend/resume. This should * only be called at bootup or when userspace isn't running. */ -void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) +void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev) { struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); u32 num_clear_entities, num_move_entities; - uint64_t size; int r, i, j; if (!adev->mman.initialized || amdgpu_in_reset(adev) || - adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu) + adev->mman.buffer_funcs_enabled || adev->gmc.is_app_apu) return; - if (enable) { - struct amdgpu_ring *ring; - struct drm_gpu_scheduler *sched; + if (!adev->mman.num_buffer_funcs_scheds) { + dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use"); + return; + } - if (!adev->mman.buffer_funcs_ring || !adev->mman.buffer_funcs_ring->sched.ready) { - dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use"); - return; - } + /* default_entity doesn't need multiple schedulers so pass only 1. */ + r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr, + &adev->mman.default_entity, + DRM_SCHED_PRIORITY_KERNEL, + adev->mman.buffer_funcs_scheds, 1, 0); + if (r < 0) { + dev_err(adev->dev, + "Failed setting up TTM entity (%d)\n", r); + return; + } - num_clear_entities = 1; - num_move_entities = 1; - ring = adev->mman.buffer_funcs_ring; - sched = &ring->sched; - r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr, - &adev->mman.default_entity, - DRM_SCHED_PRIORITY_KERNEL, - &sched, 1, 0); - if (r < 0) { - dev_err(adev->dev, - "Failed setting up TTM entity (%d)\n", r); - return; - } + num_clear_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES); + num_move_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES); - adev->mman.clear_entities = kcalloc(num_clear_entities, - sizeof(struct amdgpu_ttm_buffer_entity), - GFP_KERNEL); - atomic_set(&adev->mman.next_clear_entity, 0); - if (!adev->mman.clear_entities) - goto error_free_default_entity; + adev->mman.clear_entities = kcalloc(num_clear_entities, + sizeof(struct amdgpu_ttm_buffer_entity), + GFP_KERNEL); + atomic_set(&adev->mman.next_clear_entity, 0); + if (!adev->mman.clear_entities) + goto error_free_default_entity; - adev->mman.num_clear_entities = num_clear_entities; - - for (i = 0; i < num_clear_entities; i++) { - r = amdgpu_ttm_buffer_entity_init( - &adev->mman.gtt_mgr, &adev->mman.clear_entities[i], - DRM_SCHED_PRIORITY_NORMAL, &sched, 1, 1); - - if (r < 0) { - for (j = 0; j < i; j++) - amdgpu_ttm_buffer_entity_fini( - &adev->mman.gtt_mgr, &adev->mman.clear_entities[j]); - kfree(adev->mman.clear_entities); - adev->mman.num_clear_entities = 0; - adev->mman.clear_entities = NULL; - goto error_free_default_entity; - } + adev->mman.num_clear_entities = num_clear_entities; + + for (i = 0; i < num_clear_entities; i++) { + r = amdgpu_ttm_buffer_entity_init( + &adev->mman.gtt_mgr, + &adev->mman.clear_entities[i], + DRM_SCHED_PRIORITY_KERNEL, + adev->mman.buffer_funcs_scheds, + adev->mman.num_buffer_funcs_scheds, 1); + + if (r < 0) { + for (j = 0; j < i; j++) + amdgpu_ttm_buffer_entity_fini( + &adev->mman.gtt_mgr, &adev->mman.clear_entities[j]); + adev->mman.num_clear_entities = 0; + kfree(adev->mman.clear_entities); + goto error_free_default_entity; } + } - adev->mman.num_move_entities = num_move_entities; - atomic_set(&adev->mman.next_move_entity, 0); - for (i = 0; i < num_move_entities; i++) { - r = amdgpu_ttm_buffer_entity_init( - &adev->mman.gtt_mgr, - &adev->mman.move_entities[i], - DRM_SCHED_PRIORITY_NORMAL, &sched, 1, 2); - - if (r < 0) { - for (j = 0; j < i; j++) - amdgpu_ttm_buffer_entity_fini( - &adev->mman.gtt_mgr, &adev->mman.move_entities[j]); - adev->mman.num_move_entities = 0; - goto error_free_clear_entities; - } + adev->mman.num_move_entities = num_move_entities; + atomic_set(&adev->mman.next_move_entity, 0); + for (i = 0; i < num_move_entities; i++) { + r = amdgpu_ttm_buffer_entity_init( + &adev->mman.gtt_mgr, + &adev->mman.move_entities[i], + DRM_SCHED_PRIORITY_KERNEL, + adev->mman.buffer_funcs_scheds, + adev->mman.num_buffer_funcs_scheds, 2); + + if (r < 0) { + for (j = 0; j < i; j++) + amdgpu_ttm_buffer_entity_fini( + &adev->mman.gtt_mgr, + &adev->mman.move_entities[j]); + adev->mman.num_move_entities = 0; + goto error_free_clear_entities; } - } else { - amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr, - &adev->mman.default_entity); - for (i = 0; i < adev->mman.num_clear_entities; i++) - amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr, - &adev->mman.clear_entities[i]); - for (i = 0; i < adev->mman.num_move_entities; i++) - amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr, - &adev->mman.move_entities[i]); - /* Drop all the old fences since re-creating the scheduler entities - * will allocate new contexts. - */ - ttm_resource_manager_cleanup(man); - kfree(adev->mman.clear_entities); - adev->mman.clear_entities = NULL; - adev->mman.num_clear_entities = 0; - adev->mman.num_move_entities = 0; } /* this just adjusts TTM size idea, which sets lpfn to the correct value */ - if (enable) - size = adev->gmc.real_vram_size; - else - size = adev->gmc.visible_vram_size; - man->size = size; - adev->mman.buffer_funcs_enabled = enable; + man->size = adev->gmc.real_vram_size; + adev->mman.buffer_funcs_enabled = true; return; @@ -2405,6 +2395,42 @@ error_free_default_entity: &adev->mman.default_entity); } +/** + * amdgpu_ttm_disable_buffer_funcs - disable use of buffer functions + * + * @adev: amdgpu_device pointer + */ +void amdgpu_ttm_disable_buffer_funcs(struct amdgpu_device *adev) +{ + struct ttm_resource_manager *man = + ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); + int i; + + if (!adev->mman.buffer_funcs_enabled || amdgpu_in_reset(adev)) + return; + + amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr, + &adev->mman.default_entity); + for (i = 0; i < adev->mman.num_move_entities; i++) + amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr, + &adev->mman.move_entities[i]); + for (i = 0; i < adev->mman.num_clear_entities; i++) + amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr, + &adev->mman.clear_entities[i]); + /* Drop all the old fences since re-creating the scheduler entities + * will allocate new contexts. + */ + ttm_resource_manager_cleanup(man); + + kfree(adev->mman.clear_entities); + adev->mman.clear_entities = NULL; + adev->mman.num_clear_entities = 0; + adev->mman.num_move_entities = 0; + + man->size = adev->gmc.visible_vram_size; + adev->mman.buffer_funcs_enabled = false; +} + static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entity *entity, unsigned int num_dw, @@ -2449,7 +2475,7 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev, unsigned int i; int r; - ring = adev->mman.buffer_funcs_ring; + ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); if (!ring->sched.ready) { dev_err(adev->dev, @@ -2524,77 +2550,23 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev, } /** - * amdgpu_ttm_clear_buffer - clear memory buffers - * @bo: amdgpu buffer object - * @resv: reservation object - * @fence: dma_fence associated with the operation + * amdgpu_ttm_clear_buffer - fill a buffer with 0 + * @entity: entity to use + * @bo: the bo to fill + * @resv: fences contained in this reservation will be used as dependencies. + * @out_fence: the fence from the last clear will be stored here. It might be + * NULL if no job was run. + * @consider_clear_status: true if region reported as cleared by amdgpu_res_cleared() + * are skipped. + * @k_job_id: trace id * - * Clear the memory buffer resource. - * - * Returns: - * 0 for success or a negative error code on failure. */ -int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo, +int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity, + struct amdgpu_bo *bo, struct dma_resv *resv, - struct dma_fence **fence) -{ - struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); - struct amdgpu_ttm_buffer_entity *entity; - struct amdgpu_res_cursor cursor; - u64 addr; - int r = 0; - - if (!adev->mman.buffer_funcs_enabled) - return -EINVAL; - - if (!fence) - return -EINVAL; - entity = &adev->mman.clear_entities[0]; - *fence = dma_fence_get_stub(); - - amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor); - - mutex_lock(&entity->lock); - while (cursor.remaining) { - struct dma_fence *next = NULL; - u64 size; - - if (amdgpu_res_cleared(&cursor)) { - amdgpu_res_next(&cursor, cursor.size); - continue; - } - - /* Never clear more than 256MiB at once to avoid timeouts */ - size = min(cursor.size, 256ULL << 20); - - r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &cursor, - 0, false, &size, &addr); - if (r) - goto err; - - r = amdgpu_ttm_fill_mem(adev, entity, 0, addr, size, resv, - &next, true, - AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER); - if (r) - goto err; - - dma_fence_put(*fence); - *fence = next; - - amdgpu_res_next(&cursor, size); - } -err: - mutex_unlock(&entity->lock); - - return r; -} - -int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity, - struct amdgpu_bo *bo, - uint32_t src_data, - struct dma_resv *resv, - struct dma_fence **f, - u64 k_job_id) + struct dma_fence **out_fence, + bool consider_clear_status, + u64 k_job_id) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct dma_fence *fence = NULL; @@ -2611,6 +2583,11 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity, struct dma_fence *next; uint64_t cur_size, to; + if (consider_clear_status && amdgpu_res_cleared(&dst)) { + amdgpu_res_next(&dst, dst.size); + continue; + } + /* Never fill more than 256MiB at once to avoid timeouts */ cur_size = min(dst.size, 256ULL << 20); @@ -2620,7 +2597,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity, goto error; r = amdgpu_ttm_fill_mem(adev, entity, - src_data, to, cur_size, resv, + 0, to, cur_size, resv, &next, true, k_job_id); if (r) goto error; @@ -2632,9 +2609,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity, } error: mutex_unlock(&entity->lock); - if (f) - *f = dma_fence_get(fence); - dma_fence_put(fence); + *out_fence = fence; return r; } @@ -2682,6 +2657,41 @@ int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type) return ttm_resource_manager_evict_all(&adev->mman.bdev, man); } +void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev, + const struct amdgpu_buffer_funcs *buffer_funcs) +{ + struct drm_gpu_scheduler *sched; + struct amdgpu_vmhub *hub; + int i, n; + + adev->mman.buffer_funcs = buffer_funcs; + + for (i = 0, n = 0; i < adev->sdma.num_instances; i++) { + if (adev->sdma.has_page_queue) + sched = &adev->sdma.instance[i].page.sched; + else + sched = &adev->sdma.instance[i].ring.sched; + + if (!sched->ready) + continue; + + adev->mman.buffer_funcs_scheds[n++] = sched; + } + + if (n == 0) { + adev->mman.num_buffer_funcs_scheds = 0; + drm_warn(&adev->ddev, "No working sdma ring available\n"); + return; + } + + /* Navi1x's workaround requires us to limit to a single SDMA sched + * for ttm. + */ + hub = &adev->vmhub[AMDGPU_GFXHUB(0)]; + adev->mman.num_buffer_funcs_scheds = hub->sdma_invalidation_workaround ? + 1 : n; +} + #if defined(CONFIG_DEBUG_FS) static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index f2f23a42b3cc..b5d938b31383 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -39,7 +39,7 @@ #define AMDGPU_PL_MMIO_REMAP (TTM_PL_PRIV + 5) #define __AMDGPU_PL_NUM (TTM_PL_PRIV + 6) -#define AMDGPU_GTT_MAX_TRANSFER_SIZE 1024 +#define AMDGPU_GTT_MAX_TRANSFER_SIZE (1ULL << 22) extern const struct attribute_group amdgpu_vram_mgr_attr_group; extern const struct attribute_group amdgpu_gtt_mgr_attr_group; @@ -87,7 +87,8 @@ struct amdgpu_mman { /* buffer handling */ const struct amdgpu_buffer_funcs *buffer_funcs; - struct amdgpu_ring *buffer_funcs_ring; + struct drm_gpu_scheduler *buffer_funcs_scheds[AMDGPU_MAX_RINGS]; + u32 num_buffer_funcs_scheds; bool buffer_funcs_enabled; /* @default_entity: for workarounds, has no gart windows */ @@ -184,8 +185,8 @@ void amdgpu_ttm_unmark_vram_reserved(struct amdgpu_device *adev, int amdgpu_ttm_init(struct amdgpu_device *adev); void amdgpu_ttm_fini(struct amdgpu_device *adev); -void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, - bool enable); +void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev); +void amdgpu_ttm_disable_buffer_funcs(struct amdgpu_device *adev); int amdgpu_copy_buffer(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entity *entity, uint64_t src_offset, @@ -193,15 +194,12 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev, struct dma_resv *resv, struct dma_fence **fence, bool vm_needs_flush, uint32_t copy_flags); -int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo, +int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity, + struct amdgpu_bo *bo, struct dma_resv *resv, - struct dma_fence **fence); -int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity, - struct amdgpu_bo *bo, - uint32_t src_data, - struct dma_resv *resv, - struct dma_fence **f, - u64 k_job_id); + struct dma_fence **out_fence, + bool consider_clear_status, + u64 k_job_id); struct amdgpu_ttm_buffer_entity *amdgpu_ttm_next_clear_entity(struct amdgpu_device *adev); int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 59ffaa7b61c2..d854343b3734 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -523,6 +523,15 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que amdgpu_userq_cleanup(queue); mutex_unlock(&uq_mgr->userq_mutex); + /* + * A failed unmap means MES could not remove the hung queue and is now + * unresponsive. Recover the GPU here so the wedged MES does not fail + * the next, unrelated queue submission and trigger a reset attributed + * to an innocent workload. + */ + if (r) + queue_work(adev->reset_domain->wq, &uq_mgr->reset_work); + cancel_delayed_work_sync(&queue->hang_detect_work); uq_funcs->mqd_destroy(queue); queue->userq_mgr = NULL; @@ -680,8 +689,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) /* Update VM owner at userq submit-time for page-fault attribution. */ amdgpu_vm_set_task_info(&fpriv->vm); - r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, - GFP_KERNEL)); + r = xa_insert_irq(&adev->userq_doorbell_xa, index, queue, + GFP_KERNEL); if (r) goto clean_mqd; @@ -915,12 +924,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec, struct amdgpu_bo *bo; int ret; - spin_lock(&vm->status_lock); - while (!list_empty(&vm->invalidated)) { - bo_va = list_first_entry(&vm->invalidated, + spin_lock(&vm->individual_lock); + while (!list_empty(&vm->always_valid.evicted)) { + bo_va = list_first_entry(&vm->always_valid.evicted, struct amdgpu_bo_va, base.vm_status); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); bo = bo_va->base.bo; ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2); @@ -932,14 +941,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec, if (ret) return ret; - /* This moves the bo_va to the done list */ + /* This moves the bo_va to the idle list */ ret = amdgpu_vm_bo_update(adev, bo_va, false); if (ret) return ret; - spin_lock(&vm->status_lock); + spin_lock(&vm->individual_lock); } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); return 0; } @@ -971,7 +980,7 @@ retry_lock: if (unlikely(ret)) goto unlock_all; - ret = amdgpu_vm_lock_done_list(vm, &exec, 1); + ret = amdgpu_vm_lock_individual(vm, &exec, TTM_NUM_MOVE_FENCES + 1); drm_exec_retry_on_contention(&exec); if (unlikely(ret)) goto unlock_all; @@ -1014,7 +1023,7 @@ retry_lock: key = 0; /* Validate User Ptr BOs */ - list_for_each_entry(bo_va, &vm->done, base.vm_status) { + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) { bo = bo_va->base.bo; if (!bo) continue; @@ -1064,10 +1073,10 @@ retry_lock: /* * We need to wait for all VM updates to finish before restarting the - * queues. Using the done list like that is now ok since everything is + * queues. Using the idle list like that is now ok since everything is * locked in place. */ - list_for_each_entry(bo_va, &vm->done, base.vm_status) + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) dma_fence_wait(bo_va->last_pt_update, false); dma_fence_wait(vm->last_update, false); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index a41fb72dba94..f74ad378e407 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -593,7 +593,7 @@ free_syncobj_handles: static int amdgpu_userq_wait_count_fences(struct drm_file *filp, struct drm_amdgpu_userq_wait *wait_info, - u32 *syncobj_handles, u32 *timeline_points, + u32 *syncobj_handles, u64 *timeline_points, u32 *timeline_handles, struct drm_gem_object **gobj_write, struct drm_gem_object **gobj_read) @@ -703,7 +703,7 @@ amdgpu_userq_wait_add_fence(struct drm_amdgpu_userq_wait *wait_info, static int amdgpu_userq_wait_return_fence_info(struct drm_file *filp, struct drm_amdgpu_userq_wait *wait_info, - u32 *syncobj_handles, u32 *timeline_points, + u32 *syncobj_handles, u64 *timeline_points, u32 *timeline_handles, struct drm_gem_object **gobj_write, struct drm_gem_object **gobj_read) @@ -906,7 +906,8 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { int num_points, num_syncobj, num_read_bo_handles, num_write_bo_handles; - u32 *syncobj_handles, *timeline_points, *timeline_handles; + u32 *syncobj_handles, *timeline_handles; + u64 *timeline_points; struct drm_amdgpu_userq_wait *wait_info = data; struct drm_gem_object **gobj_write; struct drm_gem_object **gobj_read; @@ -935,7 +936,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, } ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points); - timeline_points = memdup_array_user(ptr, num_points, sizeof(u32)); + timeline_points = memdup_array_user(ptr, num_points, sizeof(u64)); if (IS_ERR(timeline_points)) { r = PTR_ERR(timeline_points); goto free_timeline_handles; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 9d5cca7da1d9..23383ac5323f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -135,7 +135,7 @@ MODULE_FIRMWARE(FIRMWARE_VEGA12); MODULE_FIRMWARE(FIRMWARE_VEGA20); static void amdgpu_uvd_idle_work_handler(struct work_struct *work); -static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo); +static void amdgpu_uvd_force_into_vcpu_segment(struct amdgpu_bo *abo); static int amdgpu_uvd_create_msg_bo_helper(struct amdgpu_device *adev, uint32_t size, @@ -158,7 +158,7 @@ static int amdgpu_uvd_create_msg_bo_helper(struct amdgpu_device *adev, amdgpu_bo_kunmap(bo); amdgpu_bo_unpin(bo); amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM); - amdgpu_uvd_force_into_uvd_segment(bo); + amdgpu_uvd_force_into_vcpu_segment(bo); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (r) goto err; @@ -188,6 +188,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) const struct common_firmware_header *hdr; unsigned int family_id; int i, j, r; + u32 vcpu_bo_domain; INIT_DELAYED_WORK(&adev->uvd.idle_work, amdgpu_uvd_idle_work_handler); @@ -319,12 +320,20 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); + /* UVD 5.0 and newer HW can use 64 bit addressing. */ + adev->uvd.address_64_bit = + !amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0); + + vcpu_bo_domain = AMDGPU_GEM_DOMAIN_VRAM; + if (adev->uvd.address_64_bit) + vcpu_bo_domain |= AMDGPU_GEM_DOMAIN_GTT; + for (j = 0; j < adev->uvd.num_uvd_inst; j++) { if (adev->uvd.harvest_config & (1 << j)) continue; + r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, + vcpu_bo_domain, &adev->uvd.inst[j].vcpu_bo, &adev->uvd.inst[j].gpu_addr, &adev->uvd.inst[j].cpu_addr); @@ -339,10 +348,6 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) adev->uvd.filp[i] = NULL; } - /* from uvd v5.0 HW addressing capacity increased to 64 bits */ - if (!amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0)) - adev->uvd.address_64_bit = true; - r = amdgpu_uvd_create_msg_bo_helper(adev, 128 << 10, &adev->uvd.ib_bo); if (r) return r; @@ -512,7 +517,7 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) } memset_io(ptr, 0, size); /* to restore uvd fence seq */ - amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring); + amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL); } } return 0; @@ -545,6 +550,24 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp) } } +static void amdgpu_uvd_force_into_vcpu_segment(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_bo *vcpu_bo = adev->uvd.inst[0].vcpu_bo; + struct amdgpu_res_cursor vcpu_cur; + + amdgpu_res_first(vcpu_bo->tbo.resource, 0, + amdgpu_bo_size(vcpu_bo), &vcpu_cur); + + bo->placement.num_placement = 1; + bo->placement.placement = &bo->placements[0]; + bo->placements[0].fpfn = ALIGN_DOWN(vcpu_cur.start, SZ_256M) >> PAGE_SHIFT; + bo->placements[0].lpfn = bo->placements[0].fpfn + (SZ_256M >> PAGE_SHIFT); + bo->placements[0].mem_type = vcpu_bo->tbo.resource->mem_type; + if (bo->placements[0].mem_type == TTM_PL_VRAM) + bo->placements[0].flags |= TTM_PL_FLAG_CONTIGUOUS; +} + static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo) { int i; @@ -595,13 +618,10 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx) if (!ctx->parser->adev->uvd.address_64_bit) { /* check if it's a message or feedback command */ cmd = amdgpu_ib_get_value(ctx->ib, ctx->idx) >> 1; - if (cmd == 0x0 || cmd == 0x3) { - /* yes, force it into VRAM */ - uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM; - - amdgpu_bo_placement_from_domain(bo, domain); - } - amdgpu_uvd_force_into_uvd_segment(bo); + if (cmd == 0x0 || cmd == 0x3) + amdgpu_uvd_force_into_vcpu_segment(bo); + else + amdgpu_uvd_force_into_uvd_segment(bo); r = ttm_bo_validate(&bo->tbo, &bo->placement, &tctx); } @@ -635,6 +655,14 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, unsigned int image_size, tmp, min_dpb_size, num_dpb_buffer; unsigned int min_ctx_size = ~0; + /* Reject invalid dimensions to prevent division by zero */ + if (width < 16 || height < 16) { + dev_WARN_ONCE(adev->dev, 1, + "Invalid UVD decoding dimensions (%dx%d)!\n", + width, height); + return -EINVAL; + } + image_size = width * height; image_size += image_size / 2; image_size = ALIGN(image_size, 1024); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index efdebd9c0a1f..eef3c9853a5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -877,9 +877,20 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, goto out; } - *size = amdgpu_ib_get_value(ib, idx + 8) * - amdgpu_ib_get_value(ib, idx + 10) * - 8 * 3 / 2; + uint32_t width, height; + width = amdgpu_ib_get_value(ib, idx + 8); + height = amdgpu_ib_get_value(ib, idx + 10); + + if (width == 0 || height == 0 || + width > 4096 || height > 2304) { + DRM_ERROR("invalid VCE image size: %ux%u\n", + width, height); + r = -EINVAL; + goto out; + } + + *size = width * height * 8 * 3 / 2; + break; case 0x04000001: /* config extension */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index debb82a2e031..616967519869 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -1486,18 +1486,27 @@ int vcn_set_powergating_state(struct amdgpu_ip_block *ip_block, } /** - * amdgpu_vcn_reset_engine - Reset a specific VCN engine - * @adev: Pointer to the AMDGPU device - * @instance_id: VCN engine instance to reset + * amdgpu_vcn_ring_reset - Reset a VCN ring + * @ring: ring to reset + * @vmid: vmid of guilty job + * @timedout_fence: fence of timed out job * + * This helper is for VCN blocks without unified queues because + * resetting the engine resets all queues in that case. With + * unified queues we have one queue per engine. * Returns: 0 on success, or a negative error code on failure. */ -static int amdgpu_vcn_reset_engine(struct amdgpu_device *adev, - uint32_t instance_id) +int amdgpu_vcn_ring_reset(struct amdgpu_ring *ring, + unsigned int vmid, + struct amdgpu_fence *timedout_fence) { - struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[instance_id]; + struct amdgpu_device *adev = ring->adev; + struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me]; int r, i; + if (adev->vcn.inst[ring->me].using_unified_queue) + return -EINVAL; + mutex_lock(&vinst->engine_reset_mutex); /* Stop the scheduler's work queue for the dec and enc rings if they are running. * This ensures that no new tasks are submitted to the queues while @@ -1519,9 +1528,13 @@ static int amdgpu_vcn_reset_engine(struct amdgpu_device *adev, if (r) goto unlock; } - amdgpu_fence_driver_force_completion(&vinst->ring_dec); + amdgpu_fence_driver_force_completion(&vinst->ring_dec, + (&vinst->ring_dec == ring) ? + &timedout_fence->base : NULL); for (i = 0; i < vinst->num_enc_rings; i++) - amdgpu_fence_driver_force_completion(&vinst->ring_enc[i]); + amdgpu_fence_driver_force_completion(&vinst->ring_enc[i], + (&vinst->ring_enc[i] == ring) ? + &timedout_fence->base : NULL); /* Restart the scheduler's work queue for the dec and enc rings * if they were stopped by this function. This allows new tasks @@ -1537,29 +1550,6 @@ unlock: return r; } -/** - * amdgpu_vcn_ring_reset - Reset a VCN ring - * @ring: ring to reset - * @vmid: vmid of guilty job - * @timedout_fence: fence of timed out job - * - * This helper is for VCN blocks without unified queues because - * resetting the engine resets all queues in that case. With - * unified queues we have one queue per engine. - * Returns: 0 on success, or a negative error code on failure. - */ -int amdgpu_vcn_ring_reset(struct amdgpu_ring *ring, - unsigned int vmid, - struct amdgpu_fence *timedout_fence) -{ - struct amdgpu_device *adev = ring->adev; - - if (adev->vcn.inst[ring->me].using_unified_queue) - return -EINVAL; - - return amdgpu_vcn_reset_engine(adev, ring->me); -} - int amdgpu_vcn_reg_dump_init(struct amdgpu_device *adev, const struct amdgpu_hwip_reg_entry *reg, u32 count) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index e8d180a412d1..9e8f7d2b898c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -283,14 +283,66 @@ unsigned int amd_sriov_msg_checksum(void *obj, return ret; } +#define AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY 512 +/* Max bad page slots allowed for SRIOV*/ +#define AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY 10665U + +/** + * amdgpu_virt_ras_realloc_eh_data_space - alloc/realloc VF bad-page @data->bps and @data->bps_bo + * @adev: amdgpu device + * @data: VF RAS error-handler data + * @pages: minimum number of new slots to add beyond @data->capacity + * + * Return: 0 on success, %-ENOMEM on failure. + */ +static int amdgpu_virt_ras_realloc_eh_data_space(struct amdgpu_device *adev, + struct amdgpu_virt_ras_err_handler_data *data, + int pages) +{ + struct eeprom_table_record *new_bps; + struct amdgpu_bo **new_bo; + unsigned int old_space; + unsigned int new_space; + unsigned int align_space; + + old_space = (unsigned int)data->capacity; + new_space = old_space + max_t(unsigned int, (unsigned int)pages, + (unsigned int)AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY); + if (new_space < old_space || new_space > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY) + return -ENOMEM; + + align_space = ALIGN(new_space, AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY); + if (align_space > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY) + return -ENOMEM; + + new_bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); + new_bo = kcalloc(align_space, sizeof(*data->bps_bo), GFP_KERNEL); + if (!new_bps || !new_bo) { + kfree(new_bps); + kfree(new_bo); + dev_warn_ratelimited(adev->dev, + "RAS WARN: failed to grow bad page table to %u slots\n", + align_space); + return -ENOMEM; + } + + memcpy(new_bps, data->bps, data->count * sizeof(*data->bps)); + memcpy(new_bo, data->bps_bo, data->count * sizeof(*data->bps_bo)); + + kfree(data->bps); + kfree(data->bps_bo); + data->bps = new_bps; + data->bps_bo = new_bo; + data->capacity = (int)align_space; + + return 0; +} + static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev) { struct amdgpu_virt *virt = &adev->virt; struct amdgpu_virt_ras_err_handler_data **data = &virt->virt_eh_data; - /* GPU will be marked bad on host if bp count more then 10, - * so alloc 512 is enough. - */ - unsigned int align_space = 512; + unsigned int align_space = AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY; void *bps = NULL; struct amdgpu_bo **bps_bo = NULL; @@ -302,12 +354,13 @@ static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev) if (!bps) goto bps_failure; - bps_bo = kmalloc_objs(*(*data)->bps_bo, align_space); + bps_bo = kcalloc(align_space, sizeof(*(*data)->bps_bo), GFP_KERNEL); if (!bps_bo) goto bps_bo_failure; (*data)->bps = bps; (*data)->bps_bo = bps_bo; + (*data)->capacity = align_space; (*data)->count = 0; (*data)->last_reserved = 0; @@ -361,17 +414,33 @@ void amdgpu_virt_release_ras_err_handler_data(struct amdgpu_device *adev) virt->virt_eh_data = NULL; } -static void amdgpu_virt_ras_add_bps(struct amdgpu_device *adev, - struct eeprom_table_record *bps, int pages) +static bool amdgpu_virt_ras_add_bps(struct amdgpu_device *adev, + const struct eeprom_table_record *bps, int pages) { struct amdgpu_virt *virt = &adev->virt; struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; + int need; - if (!data) - return; + if (!data || pages <= 0) + return false; + + if (pages > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY - data->count) { + dev_warn_ratelimited(adev->dev, + "RAS WARN: bad page table at capacity (count=%d pages=%d max=%u)\n", + data->count, pages, + AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY); + return false; + } + + need = data->count + pages; + if (need > data->capacity && + amdgpu_virt_ras_realloc_eh_data_space(adev, data, need - data->capacity)) + return false; memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps)); data->count += pages; + + return true; } static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev) @@ -443,26 +512,31 @@ static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev, memset(&bp, 0, sizeof(bp)); - if (bp_block_size) { - bp_cnt = bp_block_size / sizeof(uint64_t); - for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) { - retired_page = *(uint64_t *)(vram_usage_va + - bp_block_offset + bp_idx * sizeof(uint64_t)); - bp.retired_page = retired_page; + if (!bp_block_size) + return; - if (amdgpu_virt_ras_check_bad_page(adev, retired_page)) - continue; + bp_cnt = bp_block_size / sizeof(uint64_t); + for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) { + retired_page = *(uint64_t *)(vram_usage_va + + bp_block_offset + bp_idx * sizeof(uint64_t)); + bp.retired_page = retired_page; - amdgpu_virt_ras_add_bps(adev, &bp, 1); + if (amdgpu_virt_ras_check_bad_page(adev, retired_page)) + continue; - amdgpu_virt_ras_reserve_bps(adev); - } + if (!amdgpu_virt_ras_add_bps(adev, &bp, 1)) + break; + + amdgpu_virt_ras_reserve_bps(adev); } } static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) { struct amd_sriov_msg_pf2vf_info_header *pf2vf_info = adev->virt.fw_reserve.p_pf2vf; + struct amdgim_pf2vf_info_v1 *pf2vf_v1; + struct amd_sriov_msg_pf2vf_info *pf2vf; + uint32_t checksum; uint32_t checkval; @@ -479,7 +553,8 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) switch (pf2vf_info->version) { case 1: - checksum = ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->checksum; + pf2vf_v1 = (struct amdgim_pf2vf_info_v1 *)pf2vf_info; + checksum = pf2vf_v1->checksum; checkval = amd_sriov_msg_checksum( adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, adev->virt.fw_reserve.checksum_key, checksum); @@ -490,12 +565,12 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) return -EINVAL; } - adev->virt.gim_feature = - ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->feature_flags; + adev->virt.gim_feature = pf2vf_v1->feature_flags; break; case 2: /* TODO: missing key, need to add it later */ - checksum = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->checksum; + pf2vf = (struct amd_sriov_msg_pf2vf_info *)pf2vf_info; + checksum = pf2vf->checksum; checkval = amd_sriov_msg_checksum( adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, 0, checksum); @@ -507,11 +582,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) } adev->virt.vf2pf_update_interval_ms = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->vf2pf_update_interval_ms; - adev->virt.gim_feature = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->feature_flags.all; - adev->virt.reg_access = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->reg_access_flags.all; + pf2vf->vf2pf_update_interval_ms; + adev->virt.gim_feature = pf2vf->feature_flags.all; + adev->virt.reg_access = pf2vf->reg_access_flags.all; adev->virt.decode_max_dimension_pixels = 0; adev->virt.decode_max_frame_pixels = 0; @@ -519,26 +592,30 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) adev->virt.encode_max_frame_pixels = 0; adev->virt.is_mm_bw_enabled = false; for (i = 0; i < AMD_SRIOV_MSG_RESERVE_VCN_INST; i++) { - tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_dimension_pixels; + tmp = pf2vf->mm_bw_management[i].decode_max_dimension_pixels; adev->virt.decode_max_dimension_pixels = max(tmp, adev->virt.decode_max_dimension_pixels); - tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_frame_pixels; + tmp = pf2vf->mm_bw_management[i].decode_max_frame_pixels; adev->virt.decode_max_frame_pixels = max(tmp, adev->virt.decode_max_frame_pixels); - tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_dimension_pixels; + tmp = pf2vf->mm_bw_management[i].encode_max_dimension_pixels; adev->virt.encode_max_dimension_pixels = max(tmp, adev->virt.encode_max_dimension_pixels); - tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_frame_pixels; + tmp = pf2vf->mm_bw_management[i].encode_max_frame_pixels; adev->virt.encode_max_frame_pixels = max(tmp, adev->virt.encode_max_frame_pixels); } if ((adev->virt.decode_max_dimension_pixels > 0) || (adev->virt.encode_max_dimension_pixels > 0)) adev->virt.is_mm_bw_enabled = true; - adev->unique_id = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid; - adev->virt.ras_en_caps.all = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->ras_en_caps.all; + adev->unique_id = pf2vf->uuid; + + adev->unitid = 0; + if (amdgpu_sriov_is_unitid_support(adev)) + adev->unitid = pf2vf->unitid; + + adev->virt.ras_en_caps.all = pf2vf->ras_en_caps.all; adev->virt.ras_telemetry_en_caps.all = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->ras_telemetry_en_caps.all; + pf2vf->ras_telemetry_en_caps.all; break; default: dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version); @@ -1798,13 +1875,15 @@ amdgpu_virt_write_cpers_to_ring(struct amdgpu_device *adev, struct amd_sriov_ras_cper_dump *cper_dump = NULL; struct cper_hdr *entry = NULL; struct amdgpu_ring *ring = &adev->cper.ring_buf; - uint32_t checksum, used_size, i; + uint32_t checksum, used_size; + u64 remaining, cnt, i; int ret = 0; checksum = host_telemetry->header.checksum; used_size = host_telemetry->header.used_size; - if (used_size > (AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10)) + if (used_size < offsetof(struct amd_sriov_ras_cper_dump, buf) || + used_size > (AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10)) return -EINVAL; cper_dump = kmemdup(&host_telemetry->body.cper_dump, used_size, GFP_KERNEL); @@ -1829,11 +1908,19 @@ amdgpu_virt_write_cpers_to_ring(struct amdgpu_device *adev, } entry = (struct cper_hdr *)&cper_dump->buf[0]; + remaining = (u64)used_size - offsetof(struct amd_sriov_ras_cper_dump, buf); + cnt = min_t(u64, cper_dump->count, CPER_MAX_ALLOWED_COUNT); + + for (i = 0; i < cnt; i++) { + if (entry->record_length < sizeof(struct cper_hdr) || + entry->record_length > remaining) { + ret = -EINVAL; + goto out; + } - for (i = 0; i < cper_dump->count; i++) { amdgpu_cper_ring_write(ring, entry, entry->record_length); - entry = (struct cper_hdr *)((char *)entry + - entry->record_length); + remaining -= entry->record_length; + entry = (struct cper_hdr *)((char *)entry + entry->record_length); } if (cper_dump->overflow_count) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h index 9da0c6e9b869..d8500c3e48a1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h @@ -163,6 +163,8 @@ enum AMDGIM_FEATURE_FLAG { AMDGIM_FEATURE_RAS_CPER = (1 << 11), AMDGIM_FEATURE_XGMI_TA_EXT_PEER_LINK = (1 << 12), AMDGIM_FEATURE_XGMI_CONNECTED_TO_CPU = (1 << 13), + AMDGIM_FEATURE_PTL_SUPPORT = (1 << 14), + AMDGIM_FEATURE_UNITID_SUPPORT = (1 << 15), }; enum AMDGIM_REG_ACCESS_FLAG { @@ -263,6 +265,8 @@ struct amdgpu_virt_ras_err_handler_data { struct eeprom_table_record *bps; /* point to reserved bo array */ struct amdgpu_bo **bps_bo; + /* number of slots in bps[] / bps_bo[] (always >= count) */ + int capacity; /* the count of entries */ int count; /* last reserved entry's index + 1 */ @@ -441,6 +445,8 @@ static inline bool is_virtual_machine(void) ((adev)->virt.gim_feature & AMDGIM_FEATURE_VCN_RB_DECOUPLE) #define amdgpu_sriov_is_mes_info_enable(adev) \ ((adev)->virt.gim_feature & AMDGIM_FEATURE_MES_INFO_ENABLE) +#define amdgpu_sriov_is_unitid_support(adev) \ + ((adev)->virt.gim_feature & AMDGIM_FEATURE_UNITID_SUPPORT) #define amdgpu_virt_xgmi_migrate_enabled(adev) \ ((adev)->virt.is_xgmi_node_migrate_enabled && (adev)->gmc.xgmi.node_segment_size != 0) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c index e54295b56282..170adaf7e76a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c @@ -5,6 +5,7 @@ #include <drm/drm_simple_kms_helper.h> #include <drm/drm_gem_framebuffer_helper.h> #include <drm/drm_vblank.h> +#include <drm/drm_vblank_helper.h> #include "amdgpu.h" #ifdef CONFIG_DRM_AMDGPU_SI @@ -42,81 +43,6 @@ static const u32 amdgpu_vkms_formats[] = { DRM_FORMAT_XRGB8888, }; -static enum hrtimer_restart amdgpu_vkms_vblank_simulate(struct hrtimer *timer) -{ - struct amdgpu_crtc *amdgpu_crtc = container_of(timer, struct amdgpu_crtc, vblank_timer); - struct drm_crtc *crtc = &amdgpu_crtc->base; - struct amdgpu_vkms_output *output = drm_crtc_to_amdgpu_vkms_output(crtc); - u64 ret_overrun; - bool ret; - - ret_overrun = hrtimer_forward_now(&amdgpu_crtc->vblank_timer, - output->period_ns); - if (ret_overrun != 1) - drm_warn(amdgpu_crtc->base.dev, - "%s: vblank timer overrun count: %llu\n", - __func__, ret_overrun); - - ret = drm_crtc_handle_vblank(crtc); - /* Don't queue timer again when vblank is disabled. */ - if (!ret) - return HRTIMER_NORESTART; - - return HRTIMER_RESTART; -} - -static int amdgpu_vkms_enable_vblank(struct drm_crtc *crtc) -{ - struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); - struct amdgpu_vkms_output *out = drm_crtc_to_amdgpu_vkms_output(crtc); - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - - drm_calc_timestamping_constants(crtc, &crtc->mode); - - out->period_ns = ktime_set(0, vblank->framedur_ns); - hrtimer_start(&amdgpu_crtc->vblank_timer, out->period_ns, HRTIMER_MODE_REL); - - return 0; -} - -static void amdgpu_vkms_disable_vblank(struct drm_crtc *crtc) -{ - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - - hrtimer_try_to_cancel(&amdgpu_crtc->vblank_timer); -} - -static bool amdgpu_vkms_get_vblank_timestamp(struct drm_crtc *crtc, - int *max_error, - ktime_t *vblank_time, - bool in_vblank_irq) -{ - struct amdgpu_vkms_output *output = drm_crtc_to_amdgpu_vkms_output(crtc); - struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); - struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - - if (!READ_ONCE(vblank->enabled)) { - *vblank_time = ktime_get(); - return true; - } - - *vblank_time = READ_ONCE(amdgpu_crtc->vblank_timer.node.expires); - - if (WARN_ON(*vblank_time == vblank->time)) - return true; - - /* - * To prevent races we roll the hrtimer forward before we do any - * interrupt processing - this is how real hw works (the interrupt is - * only generated after all the vblank registers are updated) and what - * the vblank core expects. Therefore we need to always correct the - * timestampe by one frame. - */ - *vblank_time -= output->period_ns; - - return true; -} - static const struct drm_crtc_funcs amdgpu_vkms_crtc_funcs = { .set_config = drm_atomic_helper_set_config, .destroy = drm_crtc_cleanup, @@ -124,45 +50,11 @@ static const struct drm_crtc_funcs amdgpu_vkms_crtc_funcs = { .reset = drm_atomic_helper_crtc_reset, .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, - .enable_vblank = amdgpu_vkms_enable_vblank, - .disable_vblank = amdgpu_vkms_disable_vblank, - .get_vblank_timestamp = amdgpu_vkms_get_vblank_timestamp, + DRM_CRTC_VBLANK_TIMER_FUNCS, }; -static void amdgpu_vkms_crtc_atomic_enable(struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - drm_crtc_vblank_on(crtc); -} - -static void amdgpu_vkms_crtc_atomic_disable(struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - drm_crtc_vblank_off(crtc); -} - -static void amdgpu_vkms_crtc_atomic_flush(struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - unsigned long flags; - if (crtc->state->event) { - spin_lock_irqsave(&crtc->dev->event_lock, flags); - - if (drm_crtc_vblank_get(crtc) != 0) - drm_crtc_send_vblank_event(crtc, crtc->state->event); - else - drm_crtc_arm_vblank_event(crtc, crtc->state->event); - - spin_unlock_irqrestore(&crtc->dev->event_lock, flags); - - crtc->state->event = NULL; - } -} - static const struct drm_crtc_helper_funcs amdgpu_vkms_crtc_helper_funcs = { - .atomic_flush = amdgpu_vkms_crtc_atomic_flush, - .atomic_enable = amdgpu_vkms_crtc_atomic_enable, - .atomic_disable = amdgpu_vkms_crtc_atomic_disable, + DRM_CRTC_HELPER_VBLANK_FUNCS, }; static int amdgpu_vkms_crtc_init(struct drm_device *dev, struct drm_crtc *crtc, @@ -187,10 +79,6 @@ static int amdgpu_vkms_crtc_init(struct drm_device *dev, struct drm_crtc *crtc, amdgpu_crtc->pll_id = ATOM_PPLL_INVALID; amdgpu_crtc->encoder = NULL; amdgpu_crtc->connector = NULL; - amdgpu_crtc->vsync_timer_enabled = AMDGPU_IRQ_STATE_DISABLE; - - hrtimer_setup(&amdgpu_crtc->vblank_timer, &amdgpu_vkms_vblank_simulate, CLOCK_MONOTONIC, - HRTIMER_MODE_REL); return ret; } @@ -262,13 +150,13 @@ static const struct drm_plane_funcs amdgpu_vkms_plane_funcs = { }; static void amdgpu_vkms_plane_atomic_update(struct drm_plane *plane, - struct drm_atomic_state *old_state) + struct drm_atomic_commit *old_state) { return; } static int amdgpu_vkms_plane_atomic_check(struct drm_plane *plane, - struct drm_atomic_state *state) + struct drm_atomic_commit *state) { struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane); @@ -538,11 +426,6 @@ static int amdgpu_vkms_sw_init(struct amdgpu_ip_block *ip_block) static int amdgpu_vkms_sw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - int i = 0; - - for (i = 0; i < adev->mode_info.num_crtc; i++) - if (adev->mode_info.crtcs[i]) - hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer); drm_kms_helper_poll_fini(adev_to_drm(adev)); drm_mode_config_cleanup(adev_to_drm(adev)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 381901bc539f..bb99b7c3a010 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -138,177 +138,158 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm) dma_resv_assert_held(vm->root.bo->tbo.base.resv); } -/** - * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid - * - * @vm: VM to test against. - * @bo: BO to be tested. - * - * Returns true if the BO shares the dma_resv object with the root PD and is - * always guaranteed to be valid inside the VM. - */ -bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo) +/* Initialize the amdgpu_vm_bo_status object */ +static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists) { - return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv; + INIT_LIST_HEAD(&lists->evicted); + INIT_LIST_HEAD(&lists->needs_update); + INIT_LIST_HEAD(&lists->idle); } -/** - * amdgpu_vm_bo_evicted - vm_bo is evicted - * - * @vm_bo: vm_bo which is evicted - * - * State for PDs/PTs and per VM BOs which are not at the location they should - * be. +/* + * Make sure we have the lock to modify the vm_bo status and return the object + * with the status lists. */ -static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) +static struct amdgpu_vm_bo_status * +amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo) { struct amdgpu_vm *vm = vm_bo->vm; struct amdgpu_bo *bo = vm_bo->bo; - vm_bo->moved = true; - amdgpu_vm_assert_locked(vm); - spin_lock(&vm_bo->vm->status_lock); - if (bo->tbo.type == ttm_bo_type_kernel) - list_move(&vm_bo->vm_status, &vm->evicted); - else - list_move_tail(&vm_bo->vm_status, &vm->evicted); - spin_unlock(&vm_bo->vm->status_lock); -} -/** - * amdgpu_vm_bo_moved - vm_bo is moved - * - * @vm_bo: vm_bo which is moved - * - * State for per VM BOs which are moved, but that change is not yet reflected - * in the page tables. - */ -static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) -{ - amdgpu_vm_assert_locked(vm_bo->vm); - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->moved); - spin_unlock(&vm_bo->vm->status_lock); + if (amdgpu_vm_is_bo_always_valid(vm, bo)) { + /* No extra locking needed, protected by the root PD resv lock */ + amdgpu_vm_assert_locked(vm); + + if (bo->tbo.type == ttm_bo_type_kernel) + return &vm->kernel; + + return &vm->always_valid; + } + + spin_lock(&vm_bo->vm->individual_lock); + return &vm->individual; } -/** - * amdgpu_vm_bo_idle - vm_bo is idle - * - * @vm_bo: vm_bo which is now idle - * - * State for PDs/PTs and per VM BOs which have gone through the state machine - * and are now idle. - */ -static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) +/* Eventually unlock the status list lock again */ +static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo) { - amdgpu_vm_assert_locked(vm_bo->vm); - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->idle); - spin_unlock(&vm_bo->vm->status_lock); - vm_bo->moved = false; + if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo)) + amdgpu_vm_assert_locked(vm_bo->vm); + else + spin_unlock(&vm_bo->vm->individual_lock); } /** - * amdgpu_vm_bo_invalidated - vm_bo is invalidated + * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid * - * @vm_bo: vm_bo which is now invalidated + * @vm: VM to test against. + * @bo: BO to be tested. * - * State for normal BOs which are invalidated and that change not yet reflected - * in the PTs. + * Returns true if the BO shares the dma_resv object with the root PD and is + * always guaranteed to be valid inside the VM. */ -static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) +bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo) { - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); - spin_unlock(&vm_bo->vm->status_lock); + return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv; } /** - * amdgpu_vm_bo_evicted_user - vm_bo is evicted + * amdgpu_vm_bo_evicted - vm_bo is evicted * * @vm_bo: vm_bo which is evicted * - * State for BOs used by user mode queues which are not at the location they - * should be. + * State for vm_bo objects meaning the underlying BO was evicted and need to + * move in place again. */ -static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo) +static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) { + struct amdgpu_vm_bo_status *lists; + + lists = amdgpu_vm_bo_lock_lists(vm_bo); vm_bo->moved = true; - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user); - spin_unlock(&vm_bo->vm->status_lock); + list_move(&vm_bo->vm_status, &lists->evicted); + amdgpu_vm_bo_unlock_lists(vm_bo); } - /** - * amdgpu_vm_bo_relocated - vm_bo is reloacted + * amdgpu_vm_bo_needs_update - vm_bo needs pagetable update * - * @vm_bo: vm_bo which is relocated + * @vm_bo: vm_bo which is out of date * - * State for PDs/PTs which needs to update their parent PD. - * For the root PD, just move to idle state. + * State for vm_bo objects meaning the underlying BO had mapping changes (move, PRT bind/unbind) + * but the new location is not yet reflected in the page tables. */ -static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) +static void amdgpu_vm_bo_needs_update(struct amdgpu_vm_bo_base *vm_bo) { - amdgpu_vm_assert_locked(vm_bo->vm); - if (vm_bo->bo->parent) { - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); - spin_unlock(&vm_bo->vm->status_lock); + struct amdgpu_vm_bo_status *lists; + struct amdgpu_bo *bo = vm_bo->bo; + + /* + * The root PD doesn't have a parent PDE and goes directly into the + * idle state. + */ + lists = amdgpu_vm_bo_lock_lists(vm_bo); + if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) { + vm_bo->moved = false; + list_move(&vm_bo->vm_status, &lists->idle); } else { - amdgpu_vm_bo_idle(vm_bo); + list_move(&vm_bo->vm_status, &lists->needs_update); } + amdgpu_vm_bo_unlock_lists(vm_bo); } /** - * amdgpu_vm_bo_done - vm_bo is done + * amdgpu_vm_bo_idle - vm_bo is idle * - * @vm_bo: vm_bo which is now done + * @vm_bo: vm_bo which is now idle * - * State for normal BOs which are invalidated and that change has been updated - * in the PTs. + * State for vm_bo objects meaning we are done with the state machine and no + * further action is necessary. */ -static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) +static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) { - amdgpu_vm_assert_locked(vm_bo->vm); - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->done); - spin_unlock(&vm_bo->vm->status_lock); + struct amdgpu_vm_bo_status *lists; + + lists = amdgpu_vm_bo_lock_lists(vm_bo); + if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo)) + vm_bo->moved = false; + list_move(&vm_bo->vm_status, &lists->idle); + amdgpu_vm_bo_unlock_lists(vm_bo); } /** * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine * @vm: the VM which state machine to reset * - * Move all vm_bo object in the VM into a state where they will be updated - * again during validation. + * Move all vm_bo object in the VM into a state where their location will be + * updated in the page tables again. */ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm) { struct amdgpu_vm_bo_base *vm_bo, *tmp; + /* + * Don't use list splice here, we need the special handling for the root + * PD and set the moved flag appropriately. + */ amdgpu_vm_assert_locked(vm); + list_for_each_entry_safe(vm_bo, tmp, &vm->kernel.idle, vm_status) + amdgpu_vm_bo_needs_update(vm_bo); + list_for_each_entry_safe(vm_bo, tmp, &vm->always_valid.idle, vm_status) + amdgpu_vm_bo_needs_update(vm_bo); - spin_lock(&vm->status_lock); - list_splice_init(&vm->done, &vm->invalidated); - list_for_each_entry(vm_bo, &vm->invalidated, vm_status) + spin_lock(&vm->individual_lock); + list_for_each_entry_safe(vm_bo, tmp, &vm->individual.idle, vm_status) { vm_bo->moved = true; - - list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) { - struct amdgpu_bo *bo = vm_bo->bo; - - vm_bo->moved = true; - if (!bo || bo->tbo.type != ttm_bo_type_kernel) - list_move(&vm_bo->vm_status, &vm_bo->vm->moved); - else if (bo->parent) - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); + list_move(&vm_bo->vm_status, &vm->individual.needs_update); } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); } /** * amdgpu_vm_update_shared - helper to update shared memory stat * @base: base structure for tracking BO usage in a VM * - * Takes the vm status_lock and updates the shared memory stat. If the basic + * Takes the vm stats_lock and updates the shared memory stat. If the basic * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called * as well. */ @@ -321,7 +302,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base) bool shared; dma_resv_assert_held(bo->tbo.base.resv); - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base); if (base->shared != shared) { base->shared = shared; @@ -333,7 +314,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base) vm->stats[bo_memtype].drm.private += size; } } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); } /** @@ -358,11 +339,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo) * be bo->tbo.resource * @sign: if we should add (+1) or subtract (-1) from the stat * - * Caller need to have the vm status_lock held. Useful for when multiple update + * Caller need to have the vm stats_lock held. Useful for when multiple update * need to happen at the same time. */ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base, - struct ttm_resource *res, int sign) + struct ttm_resource *res, int sign) { struct amdgpu_vm *vm = base->vm; struct amdgpu_bo *bo = base->bo; @@ -386,7 +367,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base, */ if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) vm->stats[res_memtype].drm.purgeable += size; - if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype))) + if (!(bo->preferred_domains & + amdgpu_mem_type_to_domain(res_memtype))) vm->stats[bo_memtype].evicted += size; } } @@ -405,9 +387,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base, { struct amdgpu_vm *vm = base->vm; - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); amdgpu_vm_update_stats_locked(base, res, sign); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); } /** @@ -428,37 +410,34 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, base->next = NULL; INIT_LIST_HEAD(&base->vm_status); + dma_resv_assert_held(vm->root.bo->tbo.base.resv); if (!bo) return; + base->next = bo->vm_bo; bo->vm_bo = base; - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base); amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); - if (!amdgpu_vm_is_bo_always_valid(vm, bo)) + if (!amdgpu_vm_is_bo_always_valid(vm, bo)) { + amdgpu_vm_bo_idle(base); return; - - dma_resv_assert_held(vm->root.bo->tbo.base.resv); + } ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move); - if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) - amdgpu_vm_bo_relocated(base); - else - amdgpu_vm_bo_idle(base); + /* + * When a per VM isn't in the desired domain put it into the evicted + * state to make sure that it gets validated on the next best occasion. + */ if (bo->preferred_domains & amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)) - return; - - /* - * we checked all the prerequisites, but it looks like this per vm bo - * is currently evicted. add the bo to the evicted list to make sure it - * is validated on next vm use to avoid fault. - * */ - amdgpu_vm_bo_evicted(base); + amdgpu_vm_bo_needs_update(base); + else + amdgpu_vm_bo_evicted(base); } /** @@ -479,41 +458,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec, } /** - * amdgpu_vm_lock_done_list - lock all BOs on the done list + * amdgpu_vm_lock_individual - lock all BOs on the individual idle list * @vm: vm providing the BOs * @exec: drm execution context * @num_fences: number of extra fences to reserve * - * Lock the BOs on the done list in the DRM execution context. + * Lock the BOs on the individual idle list in the DRM execution context. */ -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec, - unsigned int num_fences) +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec, + unsigned int num_fences) { - struct list_head *prev = &vm->done; + struct list_head *prev = &vm->individual.idle; struct amdgpu_bo_va *bo_va; struct amdgpu_bo *bo; int ret; /* We can only trust prev->next while holding the lock */ - spin_lock(&vm->status_lock); - while (!list_is_head(prev->next, &vm->done)) { + spin_lock(&vm->individual_lock); + while (!list_is_head(prev->next, &vm->individual.idle)) { bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status); bo = bo_va->base.bo; if (bo) { amdgpu_bo_ref(bo); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); - ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1); + ret = drm_exec_prepare_obj(exec, &bo->tbo.base, num_fences); amdgpu_bo_unref(&bo); if (unlikely(ret)) return ret; - spin_lock(&vm->status_lock); + spin_lock(&vm->individual_lock); } prev = prev->next; } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); return 0; } @@ -609,10 +588,10 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, void *param) { uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm); - struct amdgpu_vm_bo_base *bo_base; - struct amdgpu_bo *bo; + struct amdgpu_vm_bo_base *bo_base, *tmp; int r; + dma_resv_assert_held(vm->root.bo->tbo.base.resv); if (vm->generation != new_vm_generation) { vm->generation = new_vm_generation; amdgpu_vm_bo_reset_state_machine(vm); @@ -622,49 +601,62 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, return r; } - spin_lock(&vm->status_lock); - while (!list_empty(&vm->evicted)) { - bo_base = list_first_entry(&vm->evicted, - struct amdgpu_vm_bo_base, - vm_status); - spin_unlock(&vm->status_lock); + list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) { + r = validate(param, bo_base->bo); + if (r) + return r; - bo = bo_base->bo; + vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo)); + bo_base->moved = true; + amdgpu_vm_bo_needs_update(bo_base); + } - r = validate(param, bo); + /* + * As soon as all page tables are in place we can start updating them + * again. + */ + amdgpu_vm_eviction_lock(vm); + vm->evicting = false; + amdgpu_vm_eviction_unlock(vm); + + list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted, + vm_status) { + r = validate(param, bo_base->bo); if (r) return r; - if (bo->tbo.type != ttm_bo_type_kernel) { - amdgpu_vm_bo_moved(bo_base); - } else { - vm->update_funcs->map_table(to_amdgpu_bo_vm(bo)); - amdgpu_vm_bo_relocated(bo_base); - } - spin_lock(&vm->status_lock); + bo_base->moved = true; + amdgpu_vm_bo_needs_update(bo_base); } - while (ticket && !list_empty(&vm->evicted_user)) { - bo_base = list_first_entry(&vm->evicted_user, - struct amdgpu_vm_bo_base, - vm_status); - spin_unlock(&vm->status_lock); - bo = bo_base->bo; - dma_resv_assert_held(bo->tbo.base.resv); + if (!ticket) + return 0; + + spin_lock(&vm->individual_lock); +restart: + list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) { + struct amdgpu_bo *bo = bo_base->bo; + + if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket) + continue; + + spin_unlock(&vm->individual_lock); r = validate(param, bo); if (r) return r; - amdgpu_vm_bo_invalidated(bo_base); + bo_base->moved = true; + amdgpu_vm_bo_needs_update(bo_base); - spin_lock(&vm->status_lock); + /* It's a bit inefficient to always jump back to the start, but + * we would need to re-structure the KFD for properly fixing + * that. + */ + spin_lock(&vm->individual_lock); + goto restart; } - spin_unlock(&vm->status_lock); - - amdgpu_vm_eviction_lock(vm); - vm->evicting = false; - amdgpu_vm_eviction_unlock(vm); + spin_unlock(&vm->individual_lock); return 0; } @@ -689,9 +681,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm) ret = !vm->evicting; amdgpu_vm_eviction_unlock(vm); - spin_lock(&vm->status_lock); - ret &= list_empty(&vm->evicted); - spin_unlock(&vm->status_lock); + ret &= list_empty(&vm->kernel.evicted); spin_lock(&vm->immediate.lock); ret &= !vm->immediate.stopped; @@ -985,18 +975,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, struct amdgpu_vm *vm, bool immediate) { struct amdgpu_vm_update_params params; - struct amdgpu_vm_bo_base *entry; + struct amdgpu_vm_bo_base *entry, *tmp; bool flush_tlb_needed = false; - LIST_HEAD(relocated); int r, idx; amdgpu_vm_assert_locked(vm); - spin_lock(&vm->status_lock); - list_splice_init(&vm->relocated, &relocated); - spin_unlock(&vm->status_lock); - - if (list_empty(&relocated)) + if (list_empty(&vm->kernel.needs_update)) return 0; if (!drm_dev_enter(adev_to_drm(adev), &idx)) @@ -1012,7 +997,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, if (r) goto error; - list_for_each_entry(entry, &relocated, vm_status) { + list_for_each_entry(entry, &vm->kernel.needs_update, vm_status) { /* vm_flush_needed after updating moved PDEs */ flush_tlb_needed |= entry->moved; @@ -1028,11 +1013,9 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, if (flush_tlb_needed) atomic64_inc(&vm->tlb_seq); - while (!list_empty(&relocated)) { - entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base, - vm_status); + list_for_each_entry_safe(entry, tmp, &vm->kernel.needs_update, + vm_status) amdgpu_vm_bo_idle(entry); - } error: drm_dev_exit(idx); @@ -1163,7 +1146,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, params.pages_addr = pages_addr; params.unlocked = unlocked; params.needs_flush = flush_tlb; - params.allow_override = allow_override; + params.override_pte = allow_override && adev->gmc.override_pte; INIT_LIST_HEAD(¶ms.tlb_flush_waitlist); amdgpu_vm_eviction_lock(vm); @@ -1260,9 +1243,9 @@ error_free: void amdgpu_vm_get_memory(struct amdgpu_vm *vm, struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]) { - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); } /** @@ -1406,7 +1389,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, else amdgpu_vm_bo_idle(&bo_va->base); } else { - amdgpu_vm_bo_done(&bo_va->base); + amdgpu_vm_bo_idle(&bo_va->base); } list_splice_init(&bo_va->invalids, &bo_va->valids); @@ -1629,31 +1612,27 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) { - struct amdgpu_bo_va *bo_va; + struct amdgpu_bo_va *bo_va, *tmp; struct dma_resv *resv; struct amdgpu_bo *bo; bool clear, unlock; int r; - spin_lock(&vm->status_lock); - while (!list_empty(&vm->moved)) { - bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, - base.vm_status); - spin_unlock(&vm->status_lock); - + list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.needs_update, + base.vm_status) { /* Per VM BOs never need to bo cleared in the page tables */ r = amdgpu_vm_bo_update(adev, bo_va, false); if (r) return r; - spin_lock(&vm->status_lock); } - while (!list_empty(&vm->invalidated)) { - bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, - base.vm_status); + spin_lock(&vm->individual_lock); + while (!list_empty(&vm->individual.needs_update)) { + bo_va = list_first_entry(&vm->individual.needs_update, + typeof(*bo_va), base.vm_status); bo = bo_va->base.bo; resv = bo->tbo.base.resv; - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); /* Try to reserve the BO to avoid clearing its ptes */ if (!adev->debug_vm && !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && @@ -1684,11 +1663,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, drm_gem_is_imported(&bo_va->base.bo->tbo.base) && (!bo_va->base.bo->tbo.resource || bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM)) - amdgpu_vm_bo_evicted_user(&bo_va->base); + amdgpu_vm_bo_evicted(&bo_va->base); - spin_lock(&vm->status_lock); + spin_lock(&vm->individual_lock); } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); return 0; } @@ -1810,7 +1789,7 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, amdgpu_vm_prt_get(adev); if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved) - amdgpu_vm_bo_moved(&bo_va->base); + amdgpu_vm_bo_needs_update(&bo_va->base); trace_amdgpu_vm_bo_map(bo_va, mapping); } @@ -2119,7 +2098,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, if (amdgpu_vm_is_bo_always_valid(vm, bo) && !before->bo_va->base.moved) - amdgpu_vm_bo_moved(&before->bo_va->base); + amdgpu_vm_bo_needs_update(&before->bo_va->base); } else { kfree(before); } @@ -2134,7 +2113,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, if (amdgpu_vm_is_bo_always_valid(vm, bo) && !after->bo_va->base.moved) - amdgpu_vm_bo_moved(&after->bo_va->base); + amdgpu_vm_bo_needs_update(&after->bo_va->base); } else { kfree(after); } @@ -2226,9 +2205,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev, } } - spin_lock(&vm->status_lock); + spin_lock(&vm->individual_lock); list_del(&bo_va->base.vm_status); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { list_del(&mapping->list); @@ -2309,13 +2288,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted) if (bo_base->moved) continue; bo_base->moved = true; - - if (bo->tbo.type == ttm_bo_type_kernel) - amdgpu_vm_bo_relocated(bo_base); - else if (amdgpu_vm_is_bo_always_valid(vm, bo)) - amdgpu_vm_bo_moved(bo_base); - else - amdgpu_vm_bo_invalidated(bo_base); + amdgpu_vm_bo_needs_update(bo_base); } } @@ -2336,10 +2309,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem, for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { struct amdgpu_vm *vm = bo_base->vm; - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1); amdgpu_vm_update_stats_locked(bo_base, new_mem, +1); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); } amdgpu_vm_bo_invalidate(bo, evicted); @@ -2487,19 +2460,6 @@ static void amdgpu_vm_destroy_task_info(struct kref *kref) kfree(ti); } -static inline struct amdgpu_vm * -amdgpu_vm_get_vm_from_pasid(struct amdgpu_device *adev, u32 pasid) -{ - struct amdgpu_vm *vm; - unsigned long flags; - - xa_lock_irqsave(&adev->vm_manager.pasids, flags); - vm = xa_load(&adev->vm_manager.pasids, pasid); - xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); - - return vm; -} - /** * amdgpu_vm_put_task_info - reference down the vm task_info ptr * @@ -2546,8 +2506,16 @@ amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm) struct amdgpu_task_info * amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid) { - return amdgpu_vm_get_task_info_vm( - amdgpu_vm_get_vm_from_pasid(adev, pasid)); + struct amdgpu_task_info *ti; + struct amdgpu_vm *vm; + unsigned long flags; + + xa_lock_irqsave(&adev->vm_manager.pasids, flags); + vm = xa_load(&adev->vm_manager.pasids, pasid); + ti = amdgpu_vm_get_task_info_vm(vm); + xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); + + return ti; } static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm) @@ -2603,16 +2571,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, vm->va = RB_ROOT_CACHED; for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) vm->reserved_vmid[i] = NULL; - INIT_LIST_HEAD(&vm->evicted); - INIT_LIST_HEAD(&vm->evicted_user); - INIT_LIST_HEAD(&vm->relocated); - INIT_LIST_HEAD(&vm->moved); - INIT_LIST_HEAD(&vm->idle); - INIT_LIST_HEAD(&vm->invalidated); - spin_lock_init(&vm->status_lock); + + amdgpu_vm_bo_status_init(&vm->kernel); + amdgpu_vm_bo_status_init(&vm->always_valid); + spin_lock_init(&vm->individual_lock); + amdgpu_vm_bo_status_init(&vm->individual); INIT_LIST_HEAD(&vm->freed); - INIT_LIST_HEAD(&vm->done); INIT_KFIFO(vm->faults); + spin_lock_init(&vm->stats_lock); r = amdgpu_vm_init_entities(adev, vm); if (r) @@ -2953,47 +2919,56 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) } /** - * amdgpu_vm_lock_by_pasid - return an amdgpu_vm and its root bo from a pasid, if possible. + * amdgpu_vm_lock_by_pasid - look up a VM by PASID and lock its root PD * @adev: amdgpu device pointer - * @root: root BO of the VM * @pasid: PASID of the VM - * The caller needs to unreserve and unref the root bo on success. + * @exec: drm_exec context to lock the root PD in + * + * Must be called from within a drm_exec_until_all_locked() loop; the caller + * runs drm_exec_retry_on_contention() afterwards. The drm_exec context holds + * a reference on the root BO until it is finalised. + * + * Return: the VM on success, or NULL if the PASID has no VM, the VM is being + * torn down, or locking the root PD failed. */ struct amdgpu_vm *amdgpu_vm_lock_by_pasid(struct amdgpu_device *adev, - struct amdgpu_bo **root, u32 pasid) + u32 pasid, struct drm_exec *exec) { unsigned long irqflags; + struct amdgpu_bo *root; struct amdgpu_vm *vm; int r; xa_lock_irqsave(&adev->vm_manager.pasids, irqflags); vm = xa_load(&adev->vm_manager.pasids, pasid); - *root = vm ? amdgpu_bo_ref(vm->root.bo) : NULL; + root = vm ? amdgpu_bo_ref(vm->root.bo) : NULL; xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags); - if (!*root) + if (!root) return NULL; - r = amdgpu_bo_reserve(*root, true); - if (r) - goto error_unref; + r = drm_exec_lock_obj(exec, &root->tbo.base); + if (r) { + amdgpu_bo_unref(&root); + return NULL; + } /* Double check that the VM still exists */ xa_lock_irqsave(&adev->vm_manager.pasids, irqflags); vm = xa_load(&adev->vm_manager.pasids, pasid); - if (vm && vm->root.bo != *root) + if (vm && vm->root.bo != root) vm = NULL; xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags); - if (!vm) - goto error_unlock; + if (!vm) { + drm_exec_unlock_obj(exec, &root->tbo.base); + amdgpu_bo_unref(&root); + return NULL; + } - return vm; -error_unlock: - amdgpu_bo_unreserve(*root); + /* The drm_exec context holds its own reference on the root BO. */ + amdgpu_bo_unref(&root); -error_unref: - amdgpu_bo_unref(root); - return NULL; + return vm; } /** @@ -3015,33 +2990,51 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, uint64_t ts, bool write_fault) { bool is_compute_context = false; - struct amdgpu_bo *root; + struct drm_exec exec; uint64_t value, flags; struct amdgpu_vm *vm; int r; - vm = amdgpu_vm_lock_by_pasid(adev, &root, pasid); - if (!vm) + drm_exec_init(&exec, 0, 1); + drm_exec_until_all_locked(&exec) { + vm = amdgpu_vm_lock_by_pasid(adev, pasid, &exec); + drm_exec_retry_on_contention(&exec); + if (!vm) + break; + } + if (!vm) { + drm_exec_fini(&exec); return false; + } is_compute_context = vm->is_compute_context; if (is_compute_context) { - /* Unreserve root since svm_range_restore_pages might try to reserve it. */ - /* TODO: rework svm_range_restore_pages so that this isn't necessary. */ - amdgpu_bo_unreserve(root); + __label__ drm_exec_retry; + + /* Release the root PD lock since svm_range_restore_pages + * might try to take it. + * TODO: rework svm_range_restore_pages so that this isn't + * necessary. + */ + drm_exec_fini(&exec); if (!svm_range_restore_pages(adev, pasid, vmid, - node_id, addr >> PAGE_SHIFT, ts, write_fault)) { - amdgpu_bo_unref(&root); + node_id, addr >> PAGE_SHIFT, ts, write_fault)) return true; - } - amdgpu_bo_unref(&root); /* Re-acquire the VM lock, could be that the VM was freed in between. */ - vm = amdgpu_vm_lock_by_pasid(adev, &root, pasid); - if (!vm) + drm_exec_init(&exec, 0, 1); + drm_exec_until_all_locked(&exec) { + vm = amdgpu_vm_lock_by_pasid(adev, pasid, &exec); + drm_exec_retry_on_contention(&exec); + if (!vm) + break; + } + if (!vm) { + drm_exec_fini(&exec); return false; + } } addr /= AMDGPU_GPU_PAGE_SIZE; @@ -3065,7 +3058,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, value = 0; } - r = dma_resv_reserve_fences(root->tbo.base.resv, 1); + r = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1); if (r) { pr_debug("failed %d to reserve fence slot\n", r); goto error_unlock; @@ -3079,110 +3072,72 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, r = amdgpu_vm_update_pdes(adev, vm, true); error_unlock: - amdgpu_bo_unreserve(root); + drm_exec_fini(&exec); if (r < 0) dev_err(adev->dev, "Can't handle page fault (%d)\n", r); - amdgpu_bo_unref(&root); - return false; } #if defined(CONFIG_DEBUG_FS) -/** - * amdgpu_debugfs_vm_bo_info - print BO info for the VM - * - * @vm: Requested VM for printing BO info - * @m: debugfs file - * - * Print BO information in debugfs file for the VM - */ -void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) -{ - struct amdgpu_bo_va *bo_va, *tmp; - u64 total_idle = 0; - u64 total_evicted = 0; - u64 total_relocated = 0; - u64 total_moved = 0; - u64 total_invalidated = 0; - u64 total_done = 0; - unsigned int total_idle_objs = 0; - unsigned int total_evicted_objs = 0; - unsigned int total_relocated_objs = 0; - unsigned int total_moved_objs = 0; - unsigned int total_invalidated_objs = 0; - unsigned int total_done_objs = 0; - unsigned int id = 0; - amdgpu_vm_assert_locked(vm); +/* print the debug info for a specific set of status lists */ +static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m, + struct amdgpu_vm_bo_status *lists) +{ + struct amdgpu_vm_bo_base *base; + unsigned int id; - spin_lock(&vm->status_lock); - seq_puts(m, "\tIdle BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - total_idle_objs = id; id = 0; - seq_puts(m, "\tEvicted BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) { - if (!bo_va->base.bo) + list_for_each_entry(base, &lists->evicted, vm_status) { + if (!base->bo) continue; - total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - total_evicted_objs = id; - id = 0; - seq_puts(m, "\tRelocated BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); + amdgpu_bo_print_info(id++, base->bo, m); } - total_relocated_objs = id; - id = 0; + id = 0; seq_puts(m, "\tMoved BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { - if (!bo_va->base.bo) + list_for_each_entry(base, &lists->needs_update, vm_status) { + if (!base->bo) continue; - total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m); + + amdgpu_bo_print_info(id++, base->bo, m); } - total_moved_objs = id; - id = 0; - seq_puts(m, "\tInvalidated BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) { - if (!bo_va->base.bo) + id = 0; + seq_puts(m, "\tIdle BOs:\n"); + list_for_each_entry(base, &lists->needs_update, vm_status) { + if (!base->bo) continue; - total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); + + amdgpu_bo_print_info(id++, base->bo, m); } - total_invalidated_objs = id; - id = 0; +} - seq_puts(m, "\tDone BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - spin_unlock(&vm->status_lock); - total_done_objs = id; - - seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle, - total_idle_objs); - seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted, - total_evicted_objs); - seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated, - total_relocated_objs); - seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved, - total_moved_objs); - seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated, - total_invalidated_objs); - seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done, - total_done_objs); +/** + * amdgpu_debugfs_vm_bo_info - print BO info for the VM + * + * @vm: Requested VM for printing BO info + * @m: debugfs file + * + * Print BO information in debugfs file for the VM + */ +void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) +{ + amdgpu_vm_assert_locked(vm); + + seq_puts(m, "\tKernel PT/PDs:\n"); + amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel); + + seq_puts(m, "\tPer VM BOs:\n"); + amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid); + + seq_puts(m, "\tIndividual BOs:\n"); + spin_lock(&vm->individual_lock); + amdgpu_debugfs_vm_bo_status_info(m, &vm->individual); + spin_unlock(&vm->individual_lock); } #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index d083d7aab75c..5822836fa4a3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -205,17 +205,35 @@ struct amdgpu_vm_bo_base { /* protected by bo being reserved */ struct amdgpu_vm_bo_base *next; - /* protected by vm status_lock */ + /* protected by vm reservation and invalidated_lock */ struct list_head vm_status; /* if the bo is counted as shared in mem stats - * protected by vm status_lock */ + * protected by vm BO being reserved */ bool shared; - /* protected by the BO being reserved */ + /* if the BO was moved and all mappings are invalid + * protected by the BO being reserved */ bool moved; }; +/* + * The following status lists contain amdgpu_vm_bo_base objects for + * either PD/PTs, per VM BOs or BOs with individual resv object. + * + * The state transits are: evicted -> needs_update -> idle + */ +struct amdgpu_vm_bo_status { + /* BOs evicted which need to move into place again */ + struct list_head evicted; + + /* BOs whose mappings changed but PDs/PTs haven't been updated */ + struct list_head needs_update; + + /* BOs done with the state machine and need no further action */ + struct list_head idle; +}; + /* provided by hw blocks that can write ptes, e.g., sdma */ struct amdgpu_vm_pte_funcs { /* number of dw to reserve per operation */ @@ -296,10 +314,10 @@ struct amdgpu_vm_update_params { bool needs_flush; /** - * @allow_override: true for memory that is not uncached: allows MTYPE - * to be overridden for NUMA local memory. + * @override_pte: true for memory that is not uncached and gmc override function is + * implemented to allow MTYPE to be overridden for NUMA local memory. */ - bool allow_override; + bool override_pte; /** * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush @@ -345,47 +363,29 @@ struct amdgpu_vm { bool evicting; unsigned int saved_flags; - /* Lock to protect vm_bo add/del/move on all lists of vm */ - spinlock_t status_lock; - - /* Memory statistics for this vm, protected by status_lock */ + /* Memory statistics for this vm, protected by stats_lock */ + spinlock_t stats_lock; struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]; + /* BO's belonging to PD/PT which are internal to the kernel. */ + struct amdgpu_vm_bo_status kernel; + /* - * The following lists contain amdgpu_vm_bo_base objects for either - * PDs, PTs or per VM BOs. The state transits are: - * - * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle + * BOs allocated by userspace where the dma_resv is shared with the + * root PD */ - - /* Per-VM and PT BOs who needs a validation */ - struct list_head evicted; - - /* PT BOs which relocated and their parent need an update */ - struct list_head relocated; - - /* per VM BOs moved, but not yet updated in the PT */ - struct list_head moved; - - /* All BOs of this VM not currently in the state machine */ - struct list_head idle; + struct amdgpu_vm_bo_status always_valid; /* * The following lists contain amdgpu_vm_bo_base objects for BOs which - * have their own dma_resv object and not depend on the root PD. Their - * state transits are: + * have their own dma_resv object and not depend on the root PD. * - * evicted_user or invalidated -> done + * Lists are protected by the individual_lock. */ + spinlock_t individual_lock; - /* BOs for user mode queues that need a validation */ - struct list_head evicted_user; - - /* regular invalidated BOs, but not yet updated in the PT */ - struct list_head invalidated; - - /* BOs which are invalidated, has been updated in the PTs */ - struct list_head done; + /* Userspace BOs with individual resv object */ + struct amdgpu_vm_bo_status individual; /* * This list contains amdgpu_bo_va_mapping objects which have been freed @@ -507,8 +507,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm); void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec, unsigned int num_fences); -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec, - unsigned int num_fences); +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec, + unsigned int num_fences); bool amdgpu_vm_ready(struct amdgpu_vm *vm); uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm); int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, @@ -593,7 +593,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, bool write_fault); struct amdgpu_vm *amdgpu_vm_lock_by_pasid(struct amdgpu_device *adev, - struct amdgpu_bo **root, u32 pasid); + u32 pasid, struct drm_exec *exec); void amdgpu_vm_set_task_info(struct amdgpu_vm *vm); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c index f078db3fef79..b31ff6f56f0d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c @@ -88,12 +88,21 @@ static int amdgpu_vm_cpu_update(struct amdgpu_vm_update_params *p, trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags, p->immediate); + if (!p->pages_addr && p->override_pte) + amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, addr, &flags); + for (i = 0; i < count; i++) { + u64 oflags = flags; + value = p->pages_addr ? amdgpu_vm_map_gart(p->pages_addr, addr) : addr; + + if (p->pages_addr && p->override_pte) + amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, value, &oflags); + amdgpu_gmc_set_pte_pde(p->adev, (void *)(uintptr_t)pe, - i, value, flags); + i, value, oflags); addr += incr; } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index a930f1522f96..e43a60d09808 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry) entry->bo->vm_bo = NULL; ttm_bo_set_bulk_move(&entry->bo->tbo, NULL); - spin_lock(&entry->vm->status_lock); list_del(&entry->vm_status); - spin_unlock(&entry->vm->status_lock); amdgpu_bo_unref(&entry->bo); } @@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params, struct amdgpu_vm_pt_cursor seek; struct amdgpu_vm_bo_base *entry; - spin_lock(¶ms->vm->status_lock); for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) { if (entry && entry->bo) list_move(&entry->vm_status, ¶ms->tlb_flush_waitlist); @@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params, /* enter start node now */ list_move(&cursor->entry->vm_status, ¶ms->tlb_flush_waitlist); - spin_unlock(¶ms->vm->status_lock); } /** @@ -710,15 +706,6 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, if (level == AMDGPU_VM_PTB) amdgpu_vm_pte_update_noretry_flags(adev, &flags); - /* APUs mapping system memory may need different MTYPEs on different - * NUMA nodes. Only do this for contiguous ranges that can be assumed - * to be on the same NUMA node. - */ - if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) && - adev->gmc.gmc_funcs->override_vm_pte_flags && - num_possible_nodes() > 1 && !params->pages_addr && params->allow_override) - amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags); - params->vm->update_funcs->update(params, pt, pe, addr, count, incr, flags); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c index 36805dcfa159..fd09a2b5a147 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c @@ -106,13 +106,13 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p, struct dma_fence **fence) { + struct drm_gpu_scheduler *sched = + container_of(p->vm->delayed.rq, typeof(*sched), rq); + struct amdgpu_ring *ring = + container_of(sched, struct amdgpu_ring, sched); struct amdgpu_ib *ib = p->job->ibs; - struct amdgpu_ring *ring; struct dma_fence *f; - ring = container_of(p->vm->delayed.rq->sched, struct amdgpu_ring, - sched); - WARN_ON(ib->length_dw == 0); amdgpu_ring_pad_ib(ring, ib); @@ -257,6 +257,9 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p, } if (!p->pages_addr) { + if (p->override_pte) + amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, addr, &flags); + /* set page commands needed */ amdgpu_vm_sdma_set_ptes(p, bo, pe, addr, count, incr, flags); @@ -275,8 +278,14 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p, p->num_dw_left -= nptes * 2; pte = (uint64_t *)&(p->job->ibs->ptr[p->num_dw_left]); for (i = 0; i < nptes; ++i, addr += incr) { + u64 oflags = flags; + pte[i] = amdgpu_vm_map_gart(p->pages_addr, addr); - pte[i] |= flags; + + if (p->override_pte) + amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, pte[i], &oflags); + + pte[i] |= oflags; } amdgpu_vm_sdma_copy_ptes(p, bo, pe, nptes); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c index f27f917e3cdb..7bf74ff93fbd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c @@ -29,6 +29,7 @@ #include "amdgpu_smu.h" #include "soc15_common.h" #include "vpe_v6_1.h" +#include "vpe_v2_0.h" #define AMDGPU_CSA_VPE_SIZE 64 /* VPE CSA resides in the 4th page of CSA */ @@ -310,6 +311,10 @@ static int vpe_early_init(struct amdgpu_ip_block *ip_block) vpe_v6_1_set_funcs(vpe); vpe->collaborate_mode = true; break; + case IP_VERSION(2, 0, 0): + case IP_VERSION(2, 2, 0): + vpe_v2_0_set_funcs(vpe); + break; default: return -EINVAL; } @@ -1014,6 +1019,19 @@ const struct amd_ip_funcs vpe_ip_funcs = { .set_powergating_state = vpe_set_powergating_state, }; +const struct amd_ip_funcs vpe2_ip_funcs = { + .name = "vpe_v2_0", + .early_init = vpe_early_init, + .sw_init = vpe_sw_init, + .sw_fini = vpe_sw_fini, + .hw_init = vpe_hw_init, + .hw_fini = vpe_hw_fini, + .suspend = vpe_suspend, + .resume = vpe_resume, + .set_clockgating_state = vpe_set_clockgating_state, + .set_powergating_state = vpe_set_powergating_state, +}; + const struct amdgpu_ip_block_version vpe_v6_1_ip_block = { .type = AMD_IP_BLOCK_TYPE_VPE, .major = 6, @@ -1021,3 +1039,11 @@ const struct amdgpu_ip_block_version vpe_v6_1_ip_block = { .rev = 0, .funcs = &vpe_ip_funcs, }; + +const struct amdgpu_ip_block_version vpe_v2_0_ip_block = { + .type = AMD_IP_BLOCK_TYPE_VPE, + .major = 2, + .minor = 0, + .rev = 0, + .funcs = &vpe2_ip_funcs, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h index 695da740a97e..5c9a9f59a02b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h @@ -107,5 +107,6 @@ int amdgpu_vpe_sysfs_reset_mask_init(struct amdgpu_device *adev); ((vpe)->funcs->load_microcode ? (vpe)->funcs->load_microcode((vpe)) : 0) extern const struct amdgpu_ip_block_version vpe_v6_1_ip_block; +extern const struct amdgpu_ip_block_version vpe_v2_0_ip_block; #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c index 42be8ee155dd..409e103ffe8c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c @@ -466,15 +466,15 @@ int amdgpu_xcp_open_device(struct amdgpu_device *adev, void amdgpu_xcp_release_sched(struct amdgpu_device *adev, struct amdgpu_ctx_entity *entity) { - struct drm_gpu_scheduler *sched; - struct amdgpu_ring *ring; + struct drm_gpu_scheduler *sched = + container_of(entity->entity.rq, typeof(*sched), rq); if (!adev->xcp_mgr) return; - sched = entity->entity.rq->sched; if (drm_sched_wqueue_ready(sched)) { - ring = to_amdgpu_ring(entity->entity.rq->sched); + struct amdgpu_ring *ring = to_amdgpu_ring(sched); + atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt); } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h index 847cfd1fd004..d80f01c0e754 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h @@ -162,7 +162,9 @@ union amd_sriov_msg_feature_flags { uint32_t ras_cper : 1; uint32_t xgmi_ta_ext_peer_link : 1; uint32_t xgmi_connected_to_cpu : 1; - uint32_t reserved : 18; + uint32_t ptl_support : 1; + uint32_t unitid_support : 1; + uint32_t reserved : 16; } flags; uint32_t all; }; @@ -256,7 +258,7 @@ struct amd_sriov_msg_pf2vf_info_header { uint32_t reserved[2]; }; -#define AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE (55) +#define AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE (59) struct amd_sriov_msg_pf2vf_info { /* header contains size and version */ struct amd_sriov_msg_pf2vf_info_header header; @@ -293,7 +295,7 @@ struct amd_sriov_msg_pf2vf_info { uint32_t vf2pf_update_interval_ms; /* identification in ROCm SMI */ uint64_t uuid; - uint32_t fcn_idx; + uint32_t pad; /* flags to indicate which register access method VF should use */ union amd_sriov_reg_access_flags reg_access_flags; /* MM BW management */ @@ -314,6 +316,13 @@ struct amd_sriov_msg_pf2vf_info { uint32_t more_bp; //Reserved for future use. union amd_sriov_ras_caps ras_en_caps; union amd_sriov_ras_caps ras_telemetry_en_caps; + /* PTL status response for guest */ + uint32_t ptl_enabled; // PTL enable status: 0=disabled, 1=enabled + uint32_t ptl_pref_format1; // Current preferred format 1 + uint32_t ptl_pref_format2; // Current preferred format 2 + /* unit ID assigned by host; vf_idx [0..254] maps to unitid [1..255] (0 = pf) */ + uint8_t unitid; + uint8_t padding[3]; //use the 3 bytes to align /* reserved */ uint32_t reserved[256 - AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE]; diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/athub_v3_0.c index d1bba9c64e16..b42d9876e0a1 100644 --- a/drivers/gpu/drm/amd/amdgpu/athub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/athub_v3_0.c @@ -43,6 +43,7 @@ static uint32_t athub_v3_0_get_cg_cntl(struct amdgpu_device *adev) data = RREG32_SOC15(ATHUB, 0, regATHUB_MISC_CNTL_V3_0_1); break; case IP_VERSION(3, 3, 0): + case IP_VERSION(3, 4, 2): data = RREG32_SOC15(ATHUB, 0, regATHUB_MISC_CNTL_V3_3_0); break; default: @@ -59,6 +60,7 @@ static void athub_v3_0_set_cg_cntl(struct amdgpu_device *adev, uint32_t data) WREG32_SOC15(ATHUB, 0, regATHUB_MISC_CNTL_V3_0_1, data); break; case IP_VERSION(3, 3, 0): + case IP_VERSION(3, 4, 2): WREG32_SOC15(ATHUB, 0, regATHUB_MISC_CNTL_V3_3_0, data); break; default: @@ -112,6 +114,7 @@ int athub_v3_0_set_clockgating(struct amdgpu_device *adev, case IP_VERSION(3, 0, 1): case IP_VERSION(3, 0, 2): case IP_VERSION(3, 3, 0): + case IP_VERSION(3, 4, 2): athub_v3_0_update_medium_grain_clock_gating(adev, state == AMD_CG_STATE_GATE); athub_v3_0_update_medium_grain_light_sleep(adev, diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c index 6e37961f6be5..ca5d091549e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/atom.c +++ b/drivers/gpu/drm/amd/amdgpu/atom.c @@ -59,6 +59,9 @@ #define ATOM_CMD_TIMEOUT_SEC 20 +/* Limit ATOM command table recursion (calltable) to avoid kernel stack overflow. */ +#define ATOM_EXECUTE_MAX_DEPTH 32 + typedef struct { struct atom_context *ctx; uint32_t *ps, *ws; @@ -1229,6 +1232,13 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, if (!base) return -EINVAL; + if (ctx->execute_depth >= ATOM_EXECUTE_MAX_DEPTH) { + DRM_ERROR("atombios command table nesting exceeded limit (%u)\n", + ATOM_EXECUTE_MAX_DEPTH); + return -ELOOP; + } + ctx->execute_depth++; + len = CU16(base + ATOM_CT_SIZE_PTR); ws = CU8(base + ATOM_CT_WS_PTR); ps = CU8(base + ATOM_CT_PS_PTR) & ATOM_CT_PS_MASK; @@ -1285,6 +1295,7 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, free: if (ws) kfree(ectx.ws); + ctx->execute_depth--; return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/atom.h b/drivers/gpu/drm/amd/amdgpu/atom.h index 825ff28731f5..bb3d9eb7eb6b 100644 --- a/drivers/gpu/drm/amd/amdgpu/atom.h +++ b/drivers/gpu/drm/amd/amdgpu/atom.h @@ -153,6 +153,9 @@ struct atom_context { uint8_t vbios_ver_str[STRLEN_NORMAL]; uint8_t date[STRLEN_NORMAL]; uint8_t build_num[STRLEN_NORMAL]; + + /* Nesting depth for ATOM_OP_CALLTABLE */ + unsigned int execute_depth; }; extern int amdgpu_atom_debug; diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index 22780c09177d..120da838ac28 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -939,7 +939,6 @@ static int cik_sdma_early_init(struct amdgpu_ip_block *ip_block) cik_sdma_set_ring_funcs(adev); cik_sdma_set_irq_funcs(adev); - cik_sdma_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &cik_sdma_vm_pte_funcs); return 0; @@ -1000,8 +999,15 @@ static int cik_sdma_sw_fini(struct amdgpu_ip_block *ip_block) static int cik_sdma_hw_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; + int r; + + r = cik_sdma_start(adev); + if (r) + return r; + + cik_sdma_set_buffer_funcs(adev); - return cik_sdma_start(adev); + return 0; } static int cik_sdma_hw_fini(struct amdgpu_ip_block *ip_block) @@ -1340,8 +1346,7 @@ static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = { static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &cik_sdma_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &cik_sdma_buffer_funcs); } const struct amdgpu_ip_block_version cik_sdma_ip_block = diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 8b60299b73ef..a9961d504833 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -4022,7 +4022,7 @@ static void gfx_v10_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -5350,6 +5350,15 @@ static void gfx_v10_0_constants_init(struct amdgpu_device *adev) gfx_v10_0_get_tcc_info(adev); adev->gfx.config.pa_sc_tile_steering_override = gfx_v10_0_init_pa_sc_tile_steering_override(adev); + /* Program DB_RING_CONTROL for multiple GFX pipes + * Default power up value is 1. + * Possible values: + * 0 - split occlusion counters between gfx pipes + * 1 - all occlusion counters to pipe 0 + * 2 - all occlusion counters to pipe 1 + */ + WREG32_FIELD15(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL, + (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1); /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ @@ -7011,6 +7020,11 @@ static int gfx_v10_0_compute_mqd_init(struct amdgpu_device *adev, void *m, tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); mqd->cp_hqd_ib_control = tmp; + tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); + mqd->cp_hqd_quantum = tmp; + /* set static priority for a compute queue/ring */ mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; @@ -7525,6 +7539,24 @@ static int gfx_v10_0_hw_init(struct amdgpu_ip_block *ip_block) if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0) && !amdgpu_sriov_vf(adev)) gfx_v10_3_set_power_brake_sequence(adev); + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; + + return 0; + +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); return r; } @@ -7534,9 +7566,9 @@ static int gfx_v10_0_hw_fini(struct amdgpu_ip_block *ip_block) cancel_delayed_work_sync(&adev->gfx.idle_work); - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); /* WA added for Vangogh asic fixing the SMU suspend failure * It needs to set power gating again during gfxoff control @@ -7832,26 +7864,6 @@ static int gfx_v10_0_early_init(struct amdgpu_ip_block *ip_block) return gfx_v10_0_init_microcode(adev); } -static int gfx_v10_0_late_init(struct amdgpu_ip_block *ip_block) -{ - struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; - - return 0; -} - static bool gfx_v10_0_is_rlc_enabled(struct amdgpu_device *adev) { uint32_t rlc_cntl; @@ -8655,7 +8667,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -8690,7 +8702,7 @@ static void gfx_v10_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -8723,9 +8735,9 @@ static void gfx_v10_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -8773,9 +8785,6 @@ static void gfx_v10_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | @@ -9800,7 +9809,6 @@ static void gfx_v10_0_ring_end_use(struct amdgpu_ring *ring) static const struct amd_ip_funcs gfx_v10_0_ip_funcs = { .name = "gfx_v10_0", .early_init = gfx_v10_0_early_init, - .late_init = gfx_v10_0_late_init, .sw_init = gfx_v10_0_sw_init, .sw_fini = gfx_v10_0_sw_fini, .hw_init = gfx_v10_0_hw_init, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index d40ab1e95480..3b12eb27a253 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -129,6 +129,18 @@ MODULE_FIRMWARE("amdgpu/gc_11_5_4_pfp.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_4_me.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_4_mec.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_4_rlc.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_5_6_pfp.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_5_6_me.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_5_6_mec.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_5_6_rlc.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_pfp.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_me.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_mec.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_rlc.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_pfp.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_me.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_mec.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_rlc.bin"); static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = { SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS), @@ -542,7 +554,7 @@ static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -1123,6 +1135,9 @@ static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->gfx.config.max_hw_contexts = 8; adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; adev->gfx.config.sc_prim_fifo_size_backend = 0x100; @@ -1606,6 +1621,9 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->gfx.me.num_me = 1; adev->gfx.me.num_pipe_per_me = 1; adev->gfx.me.num_queue_per_pipe = 2; @@ -3078,7 +3096,10 @@ static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) || amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) || amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3) || - amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4)) + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4) || + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 6) || + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 0) || + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 1)) bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1); else @@ -4417,6 +4438,11 @@ static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m, mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; + tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); + mqd->cp_hqd_quantum = tmp; + mqd->cp_hqd_active = prop->hqd_active; /* set UQ fenceaddress */ @@ -4802,6 +4828,78 @@ static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev) WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data); } +static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev, + bool enable) +{ + unsigned int irq_type; + int m, p, r; + + if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { + for (m = 0; m < adev->gfx.me.num_me; m++) { + for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_gfx; + } + } + } + } + + if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { + for (m = 0; m < adev->gfx.mec.num_mec; ++m) { + for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_compute; + } + } + } + } + + return 0; + +err_compute: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + m = adev->gfx.me.num_me; +err_gfx: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + return r; +} + static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) { int r; @@ -4899,50 +4997,31 @@ static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) if (!adev->gfx.imu_fw_version) adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0); - return r; -} + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; -static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev, - bool enable) -{ - unsigned int irq_type; - int m, p, r; + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; - if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { - for (m = 0; m < adev->gfx.me.num_me; m++) { - for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { - irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; - if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { - for (m = 0; m < adev->gfx.mec.num_mec; ++m) { - for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { - irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP - + (m * adev->gfx.mec.num_pipe_per_mec) - + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } + r = gfx_v11_0_set_userq_eop_interrupts(adev, true); + if (r) + goto err_userq_eop; return 0; + +err_userq_eop: + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); + return r; } static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block) @@ -4951,10 +5030,10 @@ static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block) cancel_delayed_work_sync(&adev->gfx.idle_work); - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); gfx_v11_0_set_userq_eop_interrupts(adev, false); + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); if (!adev->no_hw_access) { if (amdgpu_async_gfx_ring && @@ -5219,7 +5298,7 @@ static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block) /** * GFX soft reset will impact MES, need resume MES when do GFX soft reset */ - return amdgpu_mes_resume(adev); + return amdgpu_mes_resume(adev, 0); } static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev) @@ -5344,30 +5423,6 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block) return gfx_v11_0_init_microcode(adev); } -static int gfx_v11_0_late_init(struct amdgpu_ip_block *ip_block) -{ - struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; - - r = gfx_v11_0_set_userq_eop_interrupts(adev, true); - if (r) - return r; - - return 0; -} - static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev) { uint32_t rlc_cntl; @@ -5716,6 +5771,9 @@ static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1); break; default: @@ -5755,6 +5813,9 @@ static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block, case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): if (!enable) amdgpu_gfx_off_ctrl(adev, false); @@ -5790,6 +5851,9 @@ static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): gfx_v11_0_update_gfx_clock_gating(adev, state == AMD_CG_STATE_GATE); break; @@ -5953,7 +6017,7 @@ static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5988,7 +6052,7 @@ static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -6021,9 +6085,9 @@ static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -6077,9 +6141,6 @@ static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | @@ -6227,56 +6288,6 @@ static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring, ring->set_q_mode_offs = offs; } -static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring) -{ - int i, r = 0; - struct amdgpu_device *adev = ring->adev; - struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; - struct amdgpu_ring *kiq_ring = &kiq->ring; - unsigned long flags; - - if (adev->enable_mes) - return -EINVAL; - - if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) - return -EINVAL; - - spin_lock_irqsave(&kiq->ring_lock, flags); - - if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { - spin_unlock_irqrestore(&kiq->ring_lock, flags); - return -ENOMEM; - } - - /* assert preemption condition */ - amdgpu_ring_set_preempt_cond_exec(ring, false); - - /* assert IB preemption, emit the trailing fence */ - kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, - ring->trail_fence_gpu_addr, - ++ring->trail_seq); - amdgpu_ring_commit(kiq_ring); - - spin_unlock_irqrestore(&kiq->ring_lock, flags); - - /* poll the trailing fence */ - for (i = 0; i < adev->usec_timeout; i++) { - if (ring->trail_seq == - le32_to_cpu(*(ring->trail_fence_cpu_addr))) - break; - udelay(1); - } - - if (i >= adev->usec_timeout) { - r = -EINVAL; - DRM_ERROR("ring %d failed to preempt ib\n", ring->idx); - } - - /* deassert preemption condition */ - amdgpu_ring_set_preempt_cond_exec(ring, true); - return r; -} - static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume) { struct amdgpu_device *adev = ring->adev; @@ -6516,25 +6527,33 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { u32 doorbell_offset = entry->src_data[0]; - u8 me_id, pipe_id, queue_id; - struct amdgpu_ring *ring; - int i; DRM_DEBUG("IH: CP EOP\n"); - if (adev->enable_mes && doorbell_offset) { - amdgpu_userq_process_fence_irq(adev, doorbell_offset); - } else { - me_id = (entry->ring_id & 0x0c) >> 2; - pipe_id = (entry->ring_id & 0x03) >> 0; - queue_id = (entry->ring_id & 0x70) >> 4; + if (!adev->gfx.disable_kq) { + u8 me_id = (entry->ring_id & 0x0c) >> 2; + u8 pipe_id = (entry->ring_id & 0x03) >> 0; + u8 queue_id = (entry->ring_id & 0x70) >> 4; + struct amdgpu_ring *ring; + int i; switch (me_id) { case 0: - if (pipe_id == 0) - amdgpu_fence_process(&adev->gfx.gfx_ring[0]); - else - amdgpu_fence_process(&adev->gfx.gfx_ring[1]); + /* + * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, + * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). + * Require a strict (me,pipe,queue) match so userq gfx + * EOPs fall through to amdgpu_userq_process_fence_irq(). + */ + for (i = 0; i < adev->gfx.num_gfx_rings; i++) { + ring = &adev->gfx.gfx_ring[i]; + if ((ring->me == me_id) && + (ring->pipe == pipe_id) && + (ring->queue == queue_id)) { + amdgpu_fence_process(ring); + return 0; + } + } break; case 1: case 2: @@ -6546,13 +6565,20 @@ static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, */ if ((ring->me == me_id) && (ring->pipe == pipe_id) && - (ring->queue == queue_id)) + (ring->queue == queue_id)) { amdgpu_fence_process(ring); + return 0; + } } break; + default: + break; } } + if (adev->enable_mes && doorbell_offset) + amdgpu_userq_process_fence_irq(adev, doorbell_offset); + return 0; } @@ -7246,7 +7272,6 @@ static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring) static const struct amd_ip_funcs gfx_v11_0_ip_funcs = { .name = "gfx_v11_0", .early_init = gfx_v11_0_early_init, - .late_init = gfx_v11_0_late_init, .sw_init = gfx_v11_0_sw_init, .sw_fini = gfx_v11_0_sw_fini, .hw_init = gfx_v11_0_hw_init, @@ -7308,7 +7333,7 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = { .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl, .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow, .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec, - .preempt_ib = gfx_v11_0_ring_preempt_ib, + .preempt_ib = amdgpu_gfx_ring_preempt_ib, .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl, .emit_wreg = gfx_v11_0_ring_emit_wreg, .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index c35372e21261..da668a8d6abd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -440,7 +440,7 @@ static void gfx_v12_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -3275,6 +3275,11 @@ static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m, mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; + tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); + mqd->cp_hqd_quantum = tmp; + mqd->cp_hqd_active = prop->hqd_active; /* set UQ fenceaddress */ @@ -3514,10 +3519,19 @@ static int gfx_v12_0_cp_resume(struct amdgpu_device *adev) gfx_v12_0_cp_gfx_enable(adev, true); } - if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) + if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) { r = amdgpu_mes_kiq_hw_init(adev, 0); - else + /* + * With MES, GFX KIQ ring is owned by the MES and is never + * initialized/used directly by the driver, so it must + * not be left flagged as ready. mes_v12_0_hw_init() clears + * but clear here if MES init fails + */ + if (r) + adev->gfx.kiq[0].ring.sched.ready = false; + } else { r = gfx_v12_0_kiq_resume(adev); + } if (r) return r; @@ -3650,6 +3664,78 @@ static void gfx_v12_0_init_golden_registers(struct amdgpu_device *adev) } } +static int gfx_v12_0_set_userq_eop_interrupts(struct amdgpu_device *adev, + bool enable) +{ + unsigned int irq_type; + int m, p, r; + + if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { + for (m = 0; m < adev->gfx.me.num_me; m++) { + for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_gfx; + } + } + } + } + + if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { + for (m = 0; m < adev->gfx.mec.num_mec; ++m) { + for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_compute; + } + } + } + } + + return 0; + +err_compute: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + m = adev->gfx.me.num_me; +err_gfx: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + return r; +} + static int gfx_v12_0_hw_init(struct amdgpu_ip_block *ip_block) { int r; @@ -3737,50 +3823,31 @@ static int gfx_v12_0_hw_init(struct amdgpu_ip_block *ip_block) if (r) return r; - return r; -} + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; -static int gfx_v12_0_set_userq_eop_interrupts(struct amdgpu_device *adev, - bool enable) -{ - unsigned int irq_type; - int m, p, r; + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; - if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { - for (m = 0; m < adev->gfx.me.num_me; m++) { - for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { - irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; - if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { - for (m = 0; m < adev->gfx.mec.num_mec; ++m) { - for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { - irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP - + (m * adev->gfx.mec.num_pipe_per_mec) - + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } + r = gfx_v12_0_set_userq_eop_interrupts(adev, true); + if (r) + goto err_userq_eop; return 0; + +err_userq_eop: + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); + return r; } static int gfx_v12_0_hw_fini(struct amdgpu_ip_block *ip_block) @@ -3790,10 +3857,10 @@ static int gfx_v12_0_hw_fini(struct amdgpu_ip_block *ip_block) cancel_delayed_work_sync(&adev->gfx.idle_work); - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); gfx_v12_0_set_userq_eop_interrupts(adev, false); + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); if (!adev->no_hw_access) { if (amdgpu_async_gfx_ring) { @@ -3922,30 +3989,6 @@ static int gfx_v12_0_early_init(struct amdgpu_ip_block *ip_block) return gfx_v12_0_init_microcode(adev); } -static int gfx_v12_0_late_init(struct amdgpu_ip_block *ip_block) -{ - struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; - - r = gfx_v12_0_set_userq_eop_interrupts(adev, true); - if (r) - return r; - - return 0; -} - static bool gfx_v12_0_is_rlc_enabled(struct amdgpu_device *adev) { uint32_t rlc_cntl; @@ -4459,7 +4502,7 @@ static void gfx_v12_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, control |= ib->length_dw | (vmid << 24); amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -4478,7 +4521,7 @@ static void gfx_v12_0_ring_emit_ib_compute(struct amdgpu_ring *ring, u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -4509,9 +4552,9 @@ static void gfx_v12_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -4559,9 +4602,6 @@ static void gfx_v12_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | @@ -4618,56 +4658,6 @@ static unsigned gfx_v12_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, return ret; } -static int gfx_v12_0_ring_preempt_ib(struct amdgpu_ring *ring) -{ - int i, r = 0; - struct amdgpu_device *adev = ring->adev; - struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; - struct amdgpu_ring *kiq_ring = &kiq->ring; - unsigned long flags; - - if (adev->enable_mes) - return -EINVAL; - - if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) - return -EINVAL; - - spin_lock_irqsave(&kiq->ring_lock, flags); - - if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { - spin_unlock_irqrestore(&kiq->ring_lock, flags); - return -ENOMEM; - } - - /* assert preemption condition */ - amdgpu_ring_set_preempt_cond_exec(ring, false); - - /* assert IB preemption, emit the trailing fence */ - kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, - ring->trail_fence_gpu_addr, - ++ring->trail_seq); - amdgpu_ring_commit(kiq_ring); - - spin_unlock_irqrestore(&kiq->ring_lock, flags); - - /* poll the trailing fence */ - for (i = 0; i < adev->usec_timeout; i++) { - if (ring->trail_seq == - le32_to_cpu(*(ring->trail_fence_cpu_addr))) - break; - udelay(1); - } - - if (i >= adev->usec_timeout) { - r = -EINVAL; - DRM_ERROR("ring %d failed to preempt ib\n", ring->idx); - } - - /* deassert preemption condition */ - amdgpu_ring_set_preempt_cond_exec(ring, true); - return r; -} - static void gfx_v12_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t reg_val_offs) { @@ -4854,25 +4844,33 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry) { u32 doorbell_offset = entry->src_data[0]; - u8 me_id, pipe_id, queue_id; - struct amdgpu_ring *ring; - int i; DRM_DEBUG("IH: CP EOP\n"); - if (adev->enable_mes && doorbell_offset) { - amdgpu_userq_process_fence_irq(adev, doorbell_offset); - } else { - me_id = (entry->ring_id & 0x0c) >> 2; - pipe_id = (entry->ring_id & 0x03) >> 0; - queue_id = (entry->ring_id & 0x70) >> 4; + if (!adev->gfx.disable_kq) { + u8 me_id = (entry->ring_id & 0x0c) >> 2; + u8 pipe_id = (entry->ring_id & 0x03) >> 0; + u8 queue_id = (entry->ring_id & 0x70) >> 4; + struct amdgpu_ring *ring; + int i; switch (me_id) { case 0: - if (pipe_id == 0) - amdgpu_fence_process(&adev->gfx.gfx_ring[0]); - else - amdgpu_fence_process(&adev->gfx.gfx_ring[1]); + /* + * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, + * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). + * Require a strict (me,pipe,queue) match so userq gfx + * EOPs fall through to amdgpu_userq_process_fence_irq(). + */ + for (i = 0; i < adev->gfx.num_gfx_rings; i++) { + ring = &adev->gfx.gfx_ring[i]; + if ((ring->me == me_id) && + (ring->pipe == pipe_id) && + (ring->queue == queue_id)) { + amdgpu_fence_process(ring); + return 0; + } + } break; case 1: case 2: @@ -4884,13 +4882,20 @@ static int gfx_v12_0_eop_irq(struct amdgpu_device *adev, */ if ((ring->me == me_id) && (ring->pipe == pipe_id) && - (ring->queue == queue_id)) + (ring->queue == queue_id)) { amdgpu_fence_process(ring); + return 0; + } } break; + default: + break; } } + if (adev->enable_mes && doorbell_offset) + amdgpu_userq_process_fence_irq(adev, doorbell_offset); + return 0; } @@ -5485,7 +5490,6 @@ static void gfx_v12_0_ring_end_use(struct amdgpu_ring *ring) static const struct amd_ip_funcs gfx_v12_0_ip_funcs = { .name = "gfx_v12_0", .early_init = gfx_v12_0_early_init, - .late_init = gfx_v12_0_late_init, .sw_init = gfx_v12_0_sw_init, .sw_fini = gfx_v12_0_sw_fini, .hw_init = gfx_v12_0_hw_init, @@ -5538,7 +5542,7 @@ static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_gfx = { .pad_ib = amdgpu_ring_generic_pad_ib, .emit_cntxcntl = gfx_v12_0_ring_emit_cntxcntl, .init_cond_exec = gfx_v12_0_ring_emit_init_cond_exec, - .preempt_ib = gfx_v12_0_ring_preempt_ib, + .preempt_ib = amdgpu_gfx_ring_preempt_ib, .emit_wreg = gfx_v12_0_ring_emit_wreg, .emit_reg_wait = gfx_v12_0_ring_emit_reg_wait, .emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index 68db1bc73bc7..e7e9f11b9754 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -61,7 +61,7 @@ #define regCP_HQD_IB_CONTROL_DEFAULT 0x00100000 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec.bin"); -MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc.bin"); +MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc_1.bin"); #define SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0 0x00000001 #define DEFAULT_SH_MEM_CONFIG \ @@ -243,12 +243,12 @@ static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); amdgpu_ring_write(ring, /* memory (1) or register (0) */ - (WAIT_REG_MEM_MEM_SPACE(mem_space) | - WAIT_REG_MEM_OPERATION(opt) | /* wait */ - WAIT_REG_MEM_FUNCTION(3))); /* equal */ + (PACKET3_WAIT_REG_MEM__MEM_SPACE(mem_space) | + PACKET3_WAIT_REG_MEM__OPERATION(opt) | /* wait */ + PACKET3_WAIT_REG_MEM__FUNCTION(3))); /* equal */ if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -335,7 +335,7 @@ static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout) } ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); - ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; + ib.ptr[1] = PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1); ib.ptr[2] = lower_32_bits(gpu_addr); ib.ptr[3] = upper_32_bits(gpu_addr); ib.ptr[4] = 0xDEADBEEF; @@ -409,7 +409,13 @@ static int gfx_v12_1_init_microcode(struct amdgpu_device *adev) amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); if (!amdgpu_sriov_vf(adev)) { - err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) && + adev->rev_id == 0) + err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, + AMDGPU_UCODE_REQUIRED, + "amdgpu/%s_rlc_1.bin", ucode_prefix); + else + err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, AMDGPU_UCODE_REQUIRED, "amdgpu/%s_rlc.bin", ucode_prefix); if (err) @@ -712,10 +718,19 @@ static void gfx_v12_1_select_me_pipe_q(struct amdgpu_device *adev, soc_v1_0_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id)); } +#define regGFX_IMU_PARTITION_SWITCH 0x5f8c +#define regGFX_IMU_PARTITION_SWITCH_BASE_IDX 1 +#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP__SHIFT 0x2 +#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP_MASK 0x0000003CL + static int gfx_v12_1_get_xccs_per_xcp(struct amdgpu_device *adev) { - /* Fill this in when the interface is ready */ - return 1; + u32 reg_data; + + /* the register data is expected to be the same on all instances */ + reg_data = RREG32_SOC15(GC, GET_INST(GC, 0), regGFX_IMU_PARTITION_SWITCH); + + return REG_GET_FIELD(reg_data, GFX_IMU_PARTITION_SWITCH, TOTAL_XCCS_IN_XCP); } static int gfx_v12_1_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node) @@ -1136,6 +1151,7 @@ static int gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device *adev) static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block) { + uint16_t major_ver, minor_ver; int i, j, k, r, ring_id = 0; unsigned num_compute_rings; int xcc_id, num_xcc; @@ -1146,6 +1162,15 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block) adev->gfx.mec.num_mec = 1; adev->gfx.mec.num_pipe_per_mec = 4; adev->gfx.mec.num_queue_per_pipe = 8; + + if (!amdgpu_discovery_get_gc_major_minor_version( + adev, &major_ver, &minor_ver)) { + if (major_ver == 1 && minor_ver == 3) { + adev->gfx.config.max_cu_per_sh /= 2; + dev_dbg(adev->dev, "Halving max_cu_per_sh for GC Discovery table v1:3 %d\n", + adev->gfx.config.max_cu_per_sh); + } + } break; default: adev->gfx.mec.num_mec = 2; @@ -1450,6 +1475,19 @@ static void gfx_v12_1_get_tcc_info(struct amdgpu_device *adev) { } +static void gfx_v12_1_xcc_xnack_set_chicken_bits(struct amdgpu_device *adev, int xcc_id) +{ + /* NOTE: COMPRESSION_ENABLE is used a chicken bit to enable/disable xcc xnack */ + mutex_lock(&adev->srbm_mutex); + if (!adev->gmc.noretry) { + WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), + TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x1); + } else + WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), + TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x0); + mutex_unlock(&adev->srbm_mutex); +} + static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev, int xcc_id) { @@ -1477,6 +1515,8 @@ static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev, mutex_unlock(&adev->srbm_mutex); gfx_v12_1_xcc_init_compute_vmid(adev, xcc_id); + gfx_v12_1_xcc_xnack_set_chicken_bits(adev, xcc_id); + } static void gfx_v12_1_constants_init(struct amdgpu_device *adev) @@ -2246,6 +2286,11 @@ static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m, mqd->cp_mqd_stride_size = prop->mqd_stride_size ? prop->mqd_stride_size : AMDGPU_MQD_SIZE_ALIGN(adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size); + tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); + mqd->cp_hqd_quantum = tmp; + mqd->cp_hqd_active = prop->hqd_active; return 0; @@ -2502,10 +2547,19 @@ static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev, uint16_t xcc_mask gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id); - if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) + if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) { r = amdgpu_mes_kiq_hw_init(adev, xcc_id); - else + /* + * With MES, GFX KIQ ring is owned by the MES and is never + * initialized/used directly by the driver, so it must + * not be left flagged as ready. mes_v12_0_hw_init() clears + * but clear here if MES init fails + */ + if (r) + adev->gfx.kiq[xcc_id].ring.sched.ready = false; + } else { r = gfx_v12_1_xcc_kiq_resume(adev, xcc_id); + } if (r) return r; @@ -2690,6 +2744,50 @@ static void gfx_v12_1_init_golden_registers(struct amdgpu_device *adev) } } +static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev, + bool enable) +{ + unsigned int irq_type; + int m, p, r; + + if (!adev->gfx.disable_kq) + return 0; + + for (m = 0; m < adev->gfx.mec.num_mec; ++m) { + for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + + p; + if (enable) + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); + else + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + if (r) { + if (!enable) + return r; + goto err_unwind; + } + } + } + + return 0; + +err_unwind: + for (p--; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + for (m--; m >= 0; m--) { + for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + + (m * adev->gfx.mec.num_pipe_per_mec) + p; + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); + } + } + return r; +} + static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block) { int r, i, num_xcc; @@ -2758,6 +2856,24 @@ static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block) if (r) return r; + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = gfx_v12_1_set_userq_eop_interrupts(adev, true); + if (r) + goto err_userq_eop; + + return 0; + +err_userq_eop: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); return r; } @@ -2783,41 +2899,14 @@ static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev, gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id); } -static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev, - bool enable) -{ - unsigned int irq_type; - int m, p, r; - - if (adev->gfx.disable_kq) { - for (m = 0; m < adev->gfx.mec.num_mec; ++m) { - for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { - irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP - + (m * adev->gfx.mec.num_pipe_per_mec) - + p; - if (enable) - r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, - irq_type); - else - r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, - irq_type); - if (r) - return r; - } - } - } - - return 0; -} - static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; int i, num_xcc; - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); gfx_v12_1_set_userq_eop_interrupts(adev, false); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); num_xcc = NUM_XCC(adev->gfx.xcc_mask); for (i = 0; i < num_xcc; i++) { @@ -2918,26 +3007,6 @@ static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block) return gfx_v12_1_init_microcode(adev); } -static int gfx_v12_1_late_init(struct amdgpu_ip_block *ip_block) -{ - struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = gfx_v12_1_set_userq_eop_interrupts(adev, true); - if (r) - return r; - - return 0; -} - static bool gfx_v12_1_is_rlc_enabled(struct amdgpu_device *adev) { uint32_t rlc_cntl; @@ -3355,7 +3424,7 @@ static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring, uint32_t flags) { unsigned vmid = AMDGPU_JOB_GET_VMID(job); - u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); + u32 control = PACKET3_INDIRECT_BUFFER__VALID(1) | ib->length_dw | (vmid << 24); /* Currently, there is a high possibility to get wave ID mismatch * between ME and GDS, leading to a hw deadlock, because ME generates @@ -3373,7 +3442,7 @@ static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -3391,24 +3460,24 @@ static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, /* RELEASE_MEM - flush caches, send int */ amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); - amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ(1) | - PACKET3_RELEASE_MEM_GCR_GLV_WB | - PACKET3_RELEASE_MEM_GCR_GL2_WB | - PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(2) | - PACKET3_RELEASE_MEM_TEMPORAL(3) | - PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | - PACKET3_RELEASE_MEM_EVENT_INDEX(5))); - amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) | - PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0))); + amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__GCR_SEQ(1) | + PACKET3_RELEASE_MEM__GCR_GLV_WB | + PACKET3_RELEASE_MEM__GCR_GL2_WB | + PACKET3_RELEASE_MEM__GCR_GL2_SCOPE(2) | + PACKET3_RELEASE_MEM__TEMPORAL(3) | + PACKET3_RELEASE_MEM__EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | + PACKET3_RELEASE_MEM__EVENT_INDEX(5))); + amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__DATA_SEL(write64bit ? 2 : 1) | + PACKET3_RELEASE_MEM__INT_SEL(int_sel ? 2 : 0))); /* * the address should be Qword aligned if 64bit write, Dword * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -3455,12 +3524,9 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - amdgpu_ring_write(ring, (WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); + amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1))); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -3468,7 +3534,7 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, if (flags & AMDGPU_FENCE_FLAG_INT) { /* set register to trigger INT */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - amdgpu_ring_write(ring, (WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); + amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(0) | PACKET3_WRITE_DATA__WR_CONFIRM(1))); amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS)); amdgpu_ring_write(ring, 0); amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ @@ -3507,7 +3573,7 @@ static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring, cmd = (1 << 16); /* no inc addr */ break; default: - cmd = WR_CONFIRM; + cmd = PACKET3_WRITE_DATA__WR_CONFIRM(1); break; } amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); @@ -3810,12 +3876,12 @@ static int gfx_v12_1_rlc_poison_irq(struct amdgpu_device *adev, static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring) { const unsigned int gcr_cntl = - PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) | - PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) | - PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) | - PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) | - PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1) | - PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(2); + PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_INV(1) | + PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_WB(1) | + PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_INV(1) | + PACKET3_ACQUIRE_MEM__GCR_CNTL__GLK_INV(1) | + PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV(1) | + PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE(2); /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */ amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6)); @@ -3831,7 +3897,6 @@ static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring) static const struct amd_ip_funcs gfx_v12_1_ip_funcs = { .name = "gfx_v12_1", .early_init = gfx_v12_1_early_init, - .late_init = gfx_v12_1_late_init, .sw_init = gfx_v12_1_sw_init, .sw_fini = gfx_v12_1_sw_fini, .hw_init = gfx_v12_1_hw_init, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h index 21a07530c64d..77b4f6ea532f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h @@ -58,11 +58,36 @@ #define PACKET3_DISPATCH_DIRECT 0x15 #define PACKET3_DISPATCH_INDIRECT 0x16 #define PACKET3_ATOMIC_MEM 0x1E +#define PACKET3_ATOMIC_MEM__ATOMIC(x) ((((unsigned)(x)) & 0x7F) << 0) +#define PACKET3_ATOMIC_MEM__COMMAND(x) ((((unsigned)(x)) & 0xF) << 8) +#define PACKET3_ATOMIC_MEM__SCOPE(x) ((((unsigned)(x)) & 0x3) << 23) +#define PACKET3_ATOMIC_MEM__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25) +#define PACKET3_ATOMIC_MEM__ADDR_LO(x) (((unsigned)(x))) +#define PACKET3_ATOMIC_MEM__ADDR_HI(x) (((unsigned)(x))) +#define PACKET3_ATOMIC_MEM__SRC_DATA_LO(x) (((unsigned)(x))) +#define PACKET3_ATOMIC_MEM__SRC_DATA_HI(x) (((unsigned)(x))) +#define PACKET3_ATOMIC_MEM__CMP_DATA_LO(x) (((unsigned)(x))) +#define PACKET3_ATOMIC_MEM__CMP_DATA_HI(x) (((unsigned)(x))) +#define PACKET3_ATOMIC_MEM__LOOP_INTERVAL(x) ((((unsigned)(x)) & 0x1FFF) << 0) +#define PACKET3_ATOMIC_MEM__COMMAND__SINGLE_PASS_ATOMIC 0 +#define PACKET3_ATOMIC_MEM__COMMAND__LOOP_UNTIL_COMPARE_SATISFIED 1 +#define PACKET3_ATOMIC_MEM__COMMAND__WAIT_FOR_WRITE_CONFIRMATION 2 +#define PACKET3_ATOMIC_MEM__COMMAND__SEND_AND_CONTINUE 3 +#define PACKET3_ATOMIC_MEM__SCOPE__CU 0 +#define PACKET3_ATOMIC_MEM__SCOPE__SE 1 +#define PACKET3_ATOMIC_MEM__SCOPE__DEVICE 2 +#define PACKET3_ATOMIC_MEM__SCOPE__SYSTEM 3 +#define PACKET3_ATOMIC_MEM__TEMPORAL__RT 0 +#define PACKET3_ATOMIC_MEM__TEMPORAL__NT 1 +#define PACKET3_ATOMIC_MEM__TEMPORAL__FW 2 +#define PACKET3_ATOMIC_MEM__TEMPORAL__UC 3 #define PACKET3_OCCLUSION_QUERY 0x1F #define PACKET3_SET_PREDICATION 0x20 #define PACKET3_REG_RMW 0x21 #define PACKET3_COND_EXEC 0x22 #define PACKET3_PRED_EXEC 0x23 +#define PACKET3_PRED_EXEC__EXEC_COUNT(x) ((((unsigned)(x)) & 0x3FFF) << 0) +#define PACKET3_PRED_EXEC__VIRTUALXCCID_SELECT(x) ((((unsigned)(x)) & 0xFF) << 24) #define PACKET3_DRAW_INDIRECT 0x24 #define PACKET3_DRAW_INDEX_INDIRECT 0x25 #define PACKET3_INDEX_BASE 0x26 @@ -74,7 +99,11 @@ #define PACKET3_DRAW_INDEX_MULTI_AUTO 0x30 #define PACKET3_DRAW_INDEX_OFFSET_2 0x35 #define PACKET3_WRITE_DATA 0x37 -#define WRITE_DATA_DST_SEL(x) (((x) & 0xf) << 8) +#define PACKET3_WRITE_DATA__DST_SEL(x) ((((unsigned)(x)) & 0xF) << 8) +#define PACKET3_WRITE_DATA__DST_SEL__MEM_MAPPED_REGISTER 0 +#define PACKET3_WRITE_DATA__DST_SEL__TC_L2 2 +#define PACKET3_WRITE_DATA__DST_SEL__MEMORY 5 +#define PACKET3_WRITE_DATA__DST_SEL__MEMORY_MAPPED_ADC_PERSISTENT_STATE 6 /* 0 - register * 1 - reserved * 2 - tc_l2 @@ -83,27 +112,52 @@ * 5 - memory (same as tc_l2) * 6 - memory_mapped_adc_persistent_state */ -#define WRITE_DATA_SCOPE(x) (((x) & 0x3) << 12) -#define WRITE_DATA_MODE(x) (((x) & 0x3) << 14) +#define PACKET3_WRITE_DATA__SCOPE(x) ((((unsigned)(x)) & 0x3) << 12) +#define PACKET3_WRITE_DATA__SCOPE__CU 0 +#define PACKET3_WRITE_DATA__SCOPE__SE 1 +#define PACKET3_WRITE_DATA__SCOPE__DEVICE 2 +#define PACKET3_WRITE_DATA__SCOPE__SYSTEM 3 +#define PACKET3_WRITE_DATA__MODE(x) ((((unsigned)(x)) & 0x3) << 14) +#define PACKET3_WRITE_DATA__MODE__LOCAL_XCD 0 +#define PACKET3_WRITE_DATA__MODE__REMOTE_OR_LOCAL_AID 1 +#define PACKET3_WRITE_DATA__MODE__REMOTE_XCD 2 +#define PACKET3_WRITE_DATA__MODE__REMOTE_MID 3 /* 0 - local xcd * 1 - remote/local aid * 2 - remote xcd * 3 - remote mid */ -#define WRITE_DATA_ADDR_INCR (1 << 16) -#define WRITE_DATA_MID_DIE_ID(x) (((x) & 0x3) << 18) -#define WR_CONFIRM (1 << 20) -#define WRITE_DATA_XCD_DIE_ID(x) (((x) & 0xf) << 21) -#define WRITE_DATA_TEMPORAL(x) (((x) & 0x3) << 25) - /* 0 - rt - * 1 - nt - * 2 - ht - * 3 - lu - */ -#define WRITE_DATA_COOP_DISABLE (1 << 27) +#define PACKET3_WRITE_DATA__ADDR_INCR(x) ((((unsigned)(x)) & 0x1) << 16) +#define PACKET3_WRITE_DATA__ADDR_INCR__INCREMENT_ADDRESS 0 +#define PACKET3_WRITE_DATA__ADDR_INCR__DO_NOT_INCREMENT_ADDRESS 1 +#define PACKET3_WRITE_DATA__MID_DIE_ID(x) ((((unsigned)(x)) & 0x3) << 18) +#define PACKET3_WRITE_DATA__WR_CONFIRM(x) ((((unsigned)(x)) & 0x1) << 20) +#define PACKET3_WRITE_DATA__WR_CONFIRM__DO_NOT_WAIT_FOR_WRITE_CONFIRMATION 0 +#define PACKET3_WRITE_DATA__WR_CONFIRM__WAIT_FOR_WRITE_CONFIRMATION 1 +#define PACKET3_WRITE_DATA__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xF) << 21) +#define PACKET3_WRITE_DATA__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25) +#define PACKET3_WRITE_DATA__TEMPORAL__RT 0 +#define PACKET3_WRITE_DATA__TEMPORAL__NT 1 +#define PACKET3_WRITE_DATA__TEMPORAL__HT 2 +#define PACKET3_WRITE_DATA__TEMPORAL__LU 3 +#define PACKET3_WRITE_DATA__COOP_DISABLE(x) ((((unsigned)(x)) & 0x1) << 27) +#define PACKET3_WRITE_DATA__COOP_DISABLE__MASTER_AND_SLAVE_COOP 0 +#define PACKET3_WRITE_DATA__COOP_DISABLE__MASTER_ONLY 1 +#define PACKET3_WRITE_DATA__DST_MMREG_ADDR_LO(x) ((unsigned)(x)) +#define PACKET3_WRITE_DATA__DST_MEM_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2) +#define PACKET3_WRITE_DATA__DST_MMREG_ADDR_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0) +#define PACKET3_WRITE_DATA__DST_MEM_ADDR_HI(x) ((unsigned)(x)) +#define PACKET3_WRITE_DATA__DATA(x) ((unsigned)(x)) #define PACKET3_DRAW_INDEX_INDIRECT_MULTI 0x38 #define PACKET3_WAIT_REG_MEM 0x3C -#define WAIT_REG_MEM_FUNCTION(x) (((x) & 0x7) << 0) +#define PACKET3_WAIT_REG_MEM__FUNCTION(x) ((((unsigned)(x)) & 0x7) << 0) +#define PACKET3_WAIT_REG_MEM__FUNCTION__ALWAYS_PASS 0 +#define PACKET3_WAIT_REG_MEM__FUNCTION__LESS_THAN_REF_VALUE 1 +#define PACKET3_WAIT_REG_MEM__FUNCTION__LESS_THAN_EQUAL_TO_THE_REF_VALUE 2 +#define PACKET3_WAIT_REG_MEM__FUNCTION__EQUAL_TO_THE_REFERENCE_VALUE 3 +#define PACKET3_WAIT_REG_MEM__FUNCTION__NOT_EQUAL_REFERENCE_VALUE 4 +#define PACKET3_WAIT_REG_MEM__FUNCTION__GREATER_THAN_OR_EQUAL_REFERENCE_VALUE 5 +#define PACKET3_WAIT_REG_MEM__FUNCTION__GREATER_THAN_REFERENCE_VALUE 6 /* 0 - always * 1 - < * 2 - <= @@ -112,117 +166,222 @@ * 5 - >= * 6 - > */ -#define WAIT_REG_MEM_MEM_SPACE(x) (((x) & 0x3) << 4) +#define PACKET3_WAIT_REG_MEM__MEM_SPACE(x) ((((unsigned)(x)) & 0x3) << 4) +#define PACKET3_WAIT_REG_MEM__MEM_SPACE__REGISTER_SPACE 0 +#define PACKET3_WAIT_REG_MEM__MEM_SPACE__MEMORY_SPACE 1 /* 0 - reg * 1 - mem */ -#define WAIT_REG_MEM_OPERATION(x) (((x) & 0x3) << 6) +#define PACKET3_WAIT_REG_MEM__OPERATION(x) ((((unsigned)(x)) & 0x3) << 6) +#define PACKET3_WAIT_REG_MEM__OPERATION__WAIT_REG_MEM 0 +#define PACKET3_WAIT_REG_MEM__OPERATION__WR_WAIT_WR_REG 1 +#define PACKET3_WAIT_REG_MEM__OPERATION__WAIT_MEM_PREEMPTABLE 3 /* 0 - wait_reg_mem * 1 - wr_wait_wr_reg + * 2 - reserved + * 3 - wait_mem_preemptable */ -#define WAIT_REG_MEM_MODE(x) (((x) & 0x3) << 10) +#define PACKET3_WAIT_REG_MEM__MODE(x) ((((unsigned)(x)) & 0x3) << 10) +#define PACKET3_WAIT_REG_MEM__MODE__LOCAL_XCD 0 +#define PACKET3_WAIT_REG_MEM__MODE__REMOTE_OR_LOCAL_AID 1 +#define PACKET3_WAIT_REG_MEM__MODE__REMOTE_XCD 2 +#define PACKET3_WAIT_REG_MEM__MODE__REMOTE_MID 3 /* 0 - local xcd * 1 - remote/local aid * 2 - remote xcd * 3 - remote mid */ -#define WAIT_REG_MEM_MID_DIE_ID(x) (((x) & 0x3) << 12) -#define WAIT_REG_MEM_XCD_DIE_ID(x) (((x) & 0xf) << 14) -#define WAIT_REG_MEM_MES_INTR_PIPE(x) (((x) & 0x3) << 22) -#define WAIT_REG_MEM_MES_ACTION(x) (((x) & 0x1) << 24) -#define WAIT_REG_MEM_TEMPORAL(x) (((x) & 0x3) << 25) +#define PACKET3_WAIT_REG_MEM__MID_DIE_ID(x) ((((unsigned)(x)) & 0x3) << 12) +#define PACKET3_WAIT_REG_MEM__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xf) << 14) +#define PACKET3_WAIT_REG_MEM__MES_INTR_PIPE(x) ((((unsigned)(x)) & 0x3) << 22) +#define PACKET3_WAIT_REG_MEM__MES_ACTION(x) ((((unsigned)(x)) & 0x1) << 24) +#define PACKET3_WAIT_REG_MEM__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25) +#define PACKET3_WAIT_REG_MEM__TEMPORAL__RT 0 +#define PACKET3_WAIT_REG_MEM__TEMPORAL__NT 1 +#define PACKET3_WAIT_REG_MEM__TEMPORAL__HT 2 +#define PACKET3_WAIT_REG_MEM__TEMPORAL__LU 3 /* 0 - rt * 1 - nt * 2 - ht * 3 - lu */ +#define PACKET3_WAIT_REG_MEM__MEM_POLL_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2) +#define PACKET3_WAIT_REG_MEM__REG_POLL_ADDR(x) ((unsigned)(x)) +#define PACKET3_WAIT_REG_MEM__REG_WRITE_ADDR1(x) ((((unsigned)(x)) & 0x3FFFF) << 0) +#define PACKET3_WAIT_REG_MEM__MEM_POLL_ADDR_HI(x) ((unsigned)(x)) +#define PACKET3_WAIT_REG_MEM__REG_POLL_ADDR_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0) +#define PACKET3_WAIT_REG_MEM__REG_WRITE_ADDR2(x) ((((unsigned)(x)) & 0x3FFFF) << 0) +#define PACKET3_WAIT_REG_MEM__REFERENCE(x) ((unsigned)(x)) +#define PACKET3_WAIT_REG_MEM__MASK(x) ((unsigned)(x)) +#define PACKET3_WAIT_REG_MEM__POLL_INTERVAL(x) ((((unsigned)(x)) & 0xFFFF) << 0) +#define PACKET3_WAIT_REG_MEM__OPTIMIZE_ACE_OFFLOAD_MODE(x) ((((unsigned)(x)) & 0x1) << 31) #define PACKET3_INDIRECT_BUFFER 0x3F -#define INDIRECT_BUFFER_VALID (1 << 23) -#define INDIRECT_BUFFER_TEMPORAL(x) (x) << 28) - /* 0 - rt - * 1 - nt - * 2 - ht - * 3 - lu - */ +#define PACKET3_INDIRECT_BUFFER__IB_BASE_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2) +#define PACKET3_INDIRECT_BUFFER__IB_BASE_HI(x) ((unsigned)(x)) +#define PACKET3_INDIRECT_BUFFER__IB_SIZE(x) ((((unsigned)(x)) & 0xFFFFF) << 0) +#define PACKET3_INDIRECT_BUFFER__CHAIN(x) ((((unsigned)(x)) & 0x1) << 20) +#define PACKET3_INDIRECT_BUFFER__OFFLOAD_POLLING(x) ((((unsigned)(x)) & 0x1) << 21) +#define PACKET3_INDIRECT_BUFFER__VALID(x) ((((unsigned)(x)) & 0x1) << 23) +#define PACKET3_INDIRECT_BUFFER__VMID(x) ((((unsigned)(x)) & 0xF) << 24) +#define PACKET3_INDIRECT_BUFFER__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 28) +#define PACKET3_INDIRECT_BUFFER__INHERIT_VMID(x) ((((unsigned)(x)) & 0x1) << 30) +#define PACKET3_INDIRECT_BUFFER__PRIV(x) ((((unsigned)(x)) & 0x1) << 31) +#define PACKET3_INDIRECT_BUFFER__TEMPORAL__RT 0 +#define PACKET3_INDIRECT_BUFFER__TEMPORAL__NT 1 +#define PACKET3_INDIRECT_BUFFER__TEMPORAL__HT 2 +#define PACKET3_INDIRECT_BUFFER__TEMPORAL__LU 3 #define PACKET3_COND_INDIRECT_BUFFER 0x3F #define PACKET3_COPY_DATA 0x40 -#define COPY_DATA_SRC_SEL(x) (((x) & 0xf) << 0) -#define COPY_DATA_DST_SEL(x) (((x) & 0xf) << 8) -#define COPY_DATA_SRC_SCOPE(x) (((x) & 0x3) << 4) -#define COPY_DATA_DST_SCOPE(x) (((x) & 0x3) << 27) -#define COPY_DATA_MODE(x) (((x) & 0x3) << 6) - /* 0 - local xcd - * 1 - remote/local aid - * 2 - remote xcd - * 3 - remote mid - */ -#define COPY_DATA_SRC_TEMPORAL(x) (((x) & 0x3) << 13) -#define COPY_DATA_DST_TEMPORAL(x) (((x) & 0x3) << 25) - /* 0 - rt - * 1 - nt - * 2 - ht - * 3 - lu - */ -#define COPY_DATA_COUNT_SEL (1 << 16) -#define COPY_DATA_SRC_DST_REMOTE_MODE(x) (((x)) & 0x1 << 16) - /* 0 - src remote - * 1 - dst remote - */ -#define COPY_DATA_MID_DIE_ID(x) (((x) & 0x3) << 18) -#define COPY_DATA_XCD_DIE_ID(x) (((x) & 0xf) << 21) -#define COPY_DATA_PQ_EXE_STATUS (1 << 27) +#define PACKET3_COPY_DATA__SRC_SEL(x) ((((unsigned)(x)) & 0xF) << 0) +#define PACKET3_COPY_DATA__SRC_SCOPE(x) ((((unsigned)(x)) & 0x3) << 4) +#define PACKET3_COPY_DATA__MODE(x) ((((unsigned)(x)) & 0x3) << 6) +#define PACKET3_COPY_DATA__DST_SEL(x) ((((unsigned)(x)) & 0xF) << 8) +#define PACKET3_COPY_DATA__SRC_TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 13) +#define PACKET3_COPY_DATA__COUNT_SEL(x) ((((unsigned)(x)) & 0x1) << 16) +#define PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE(x) ((((unsigned)(x)) & 0x1) << 17) +#define PACKET3_COPY_DATA__MID_DIE_ID(x) ((((unsigned)(x)) & 0x3) << 18) +#define PACKET3_COPY_DATA__WR_CONFIRM(x) ((((unsigned)(x)) & 0x1) << 20) +#define PACKET3_COPY_DATA__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xF) << 21) +#define PACKET3_COPY_DATA__DST_TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25) +#define PACKET3_COPY_DATA__DST_SCOPE(x) ((((unsigned)(x)) & 0x3) << 27) +#define PACKET3_COPY_DATA__PQ_EXE_STATUS(x) ((((unsigned)(x)) & 0x1) << 29) +#define PACKET3_COPY_DATA__SRC_REG_OFFSET_LO(x) ((unsigned)(x)) +#define PACKET3_COPY_DATA__SRC_32B_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2) +#define PACKET3_COPY_DATA__SRC_64B_ADDR_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3) +#define PACKET3_COPY_DATA__IMM_DATA(x) ((unsigned)(x)) +#define PACKET3_COPY_DATA__SRC_REG_OFFSET_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0) +#define PACKET3_COPY_DATA__SRC_MEMTC_ADDR_HI(x) ((unsigned)(x)) +#define PACKET3_COPY_DATA__SRC_IMM_DATA(x) ((unsigned)(x)) +#define PACKET3_COPY_DATA__DST_REG_OFFSET_LO(x) ((unsigned)(x)) +#define PACKET3_COPY_DATA__DST_32B_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2) +#define PACKET3_COPY_DATA__DST_64B_ADDR_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3) +#define PACKET3_COPY_DATA__DST_REG_OFFSET_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0) +#define PACKET3_COPY_DATA__DST_ADDR_HI(x) ((unsigned)(x)) +#define PACKET3_COPY_DATA__SRC_SEL__MEM_MAPPED_REGISTER 0 +#define PACKET3_COPY_DATA__SRC_SEL__TC_L2_OBSOLETE 1 +#define PACKET3_COPY_DATA__SRC_SEL__TC_L2 2 +#define PACKET3_COPY_DATA__SRC_SEL__PERFCOUNTERS 4 +#define PACKET3_COPY_DATA__SRC_SEL__IMMEDIATE_DATA 5 +#define PACKET3_COPY_DATA__SRC_SEL__ATOMIC_RETURN_DATA 6 +#define PACKET3_COPY_DATA__SRC_SEL__GPU_CLOCK_COUNT 9 +#define PACKET3_COPY_DATA__SRC_SEL__SYSTEM_CLOCK_COUNT 10 +#define PACKET3_COPY_DATA__SRC_SCOPE__CU 0 +#define PACKET3_COPY_DATA__SRC_SCOPE__SE 1 +#define PACKET3_COPY_DATA__SRC_SCOPE__DEVICE 2 +#define PACKET3_COPY_DATA__SRC_SCOPE__SYSTEM 3 +#define PACKET3_COPY_DATA__MODE__LOCAL_XCD 0 +#define PACKET3_COPY_DATA__MODE__REMOTE_OR_LOCAL_AID 1 +#define PACKET3_COPY_DATA__MODE__REMOTE_XCD 2 +#define PACKET3_COPY_DATA__MODE__REMOTE_MID 3 +#define PACKET3_COPY_DATA__DST_SEL__MEM_MAPPED_REGISTER 0 +#define PACKET3_COPY_DATA__DST_SEL__TC_L2 2 +#define PACKET3_COPY_DATA__DST_SEL__PERFCOUNTERS 4 +#define PACKET3_COPY_DATA__DST_SEL__TC_L2_OBSOLETE 5 +#define PACKET3_COPY_DATA__DST_SEL__MEM_MAPPED_REG_DC 6 +#define PACKET3_COPY_DATA__SRC_TEMPORAL__RT 0 +#define PACKET3_COPY_DATA__SRC_TEMPORAL__NT 1 +#define PACKET3_COPY_DATA__SRC_TEMPORAL__HT 2 +#define PACKET3_COPY_DATA__SRC_TEMPORAL__LU 3 +#define PACKET3_COPY_DATA__COUNT_SEL__32_BITS_OF_DATA 0 +#define PACKET3_COPY_DATA__COUNT_SEL__64_BITS_OF_DATA 1 +#define PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE__SRC_IS_REMOTE 0 +#define PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE__DST_IS_REMOTE 1 +#define PACKET3_COPY_DATA__WR_CONFIRM__DO_NOT_WAIT_FOR_CONFIRMATION 0 +#define PACKET3_COPY_DATA__WR_CONFIRM__WAIT_FOR_CONFIRMATION 1 +#define PACKET3_COPY_DATA__DST_TEMPORAL__RT 0 +#define PACKET3_COPY_DATA__DST_TEMPORAL__NT 1 +#define PACKET3_COPY_DATA__DST_TEMPORAL__HT 2 +#define PACKET3_COPY_DATA__DST_TEMPORAL__LU 3 +#define PACKET3_COPY_DATA__DST_SCOPE__CU 0 +#define PACKET3_COPY_DATA__DST_SCOPE__SE 1 +#define PACKET3_COPY_DATA__DST_SCOPE__DEVICE 2 +#define PACKET3_COPY_DATA__DST_SCOPE__SYSTEM 3 +#define PACKET3_COPY_DATA__PQ_EXE_STATUS__DEFAULT 0 +#define PACKET3_COPY_DATA__PQ_EXE_STATUS__PHASE_UPDATE 1 #define PACKET3_PFP_SYNC_ME 0x42 #define PACKET3_COND_WRITE 0x45 #define PACKET3_EVENT_WRITE 0x46 -#define EVENT_TYPE(x) ((x) << 0) -#define EVENT_INDEX(x) ((x) << 8) - /* 0 - any non-TS event - * 1 - ZPASS_DONE, PIXEL_PIPE_STAT_* - * 2 - SAMPLE_PIPELINESTAT - * 3 - SAMPLE_STREAMOUTSTAT* - * 4 - *S_PARTIAL_FLUSH - */ +#define PACKET3_EVENT_WRITE__EVENT_TYPE(x) ((((unsigned)(x)) & 0x3F) << 0) +#define PACKET3_EVENT_WRITE__EVENT_INDEX(x) ((((unsigned)(x)) & 0xF) << 8) +#define PACKET3_EVENT_WRITE__OFFLOAD_ENABLE(x) ((((unsigned)(x)) & 0x1) << 31) +#define PACKET3_EVENT_WRITE__ADDRESS_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3) +#define PACKET3_EVENT_WRITE__ADDRESS_HI(x) ((unsigned)(x)) +#define PACKET3_EVENT_WRITE__EVENT_INDEX__OTHER 0 +#define PACKET3_EVENT_WRITE__EVENT_INDEX__SAMPLE_PIPELINESTAT 2 +#define PACKET3_EVENT_WRITE__EVENT_INDEX__CS_PARTIAL_FLUSH 4 +#define PACKET3_EVENT_WRITE_EOP 0x47 +#define PACKET3_EVENT_WRITE_EOS 0x48 #define PACKET3_RELEASE_MEM 0x49 -#define PACKET3_RELEASE_MEM_EVENT_TYPE(x) ((x) << 0) -#define PACKET3_RELEASE_MEM_EVENT_INDEX(x) ((x) << 8) -#define PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(x) ((x) << 12) -#define PACKET3_RELEASE_MEM_GCR_GLV_INV (1 << 14) -#define PACKET3_RELEASE_MEM_GCR_GL2_US (1 << 16) -#define PACKET3_RELEASE_MEM_GCR_GL2_RANGE(x) ((x) << 17) -#define PACKET3_RELEASE_MEM_GCR_GL2_DISCARD (1 << 19) -#define PACKET3_RELEASE_MEM_GCR_GL2_INV (1 << 20) -#define PACKET3_RELEASE_MEM_GCR_GL2_WB (1 << 21) -#define PACKET3_RELEASE_MEM_GCR_SEQ(x) ((x) << 22) -#define PACKET3_RELEASE_MEM_GCR_GLV_WB (1 << 24) -#define PACKET3_RELEASE_MEM_TEMPORAL(x) ((x) << 25) +#define PACKET3_RELEASE_MEM__EVENT_TYPE(x) ((((unsigned)(x)) & 0x3F) << 0) +#define PACKET3_RELEASE_MEM__WAIT_SYNC(x) ((((unsigned)(x)) & 0x1) << 7) +#define PACKET3_RELEASE_MEM__EVENT_INDEX(x) ((((unsigned)(x)) & 0xF) << 8) +#define PACKET3_RELEASE_MEM__EVENT_INDEX__END_OF_PIPE 5 +#define PACKET3_RELEASE_MEM__EVENT_INDEX__SHADER_DONE 6 +#define PACKET3_RELEASE_MEM__GCR_CNTL(x) ((((unsigned)(x)) & 0x1FFF) << 12) +#define PACKET3_RELEASE_MEM__GCR_GL2_SCOPE(x) ((x) << 12) +#define PACKET3_RELEASE_MEM__GCR_GLV_INV (1 << 14) +#define PACKET3_RELEASE_MEM__GCR_GL2_US (1 << 16) +#define PACKET3_RELEASE_MEM__GCR_GL2_RANGE(x) ((x) << 17) +#define PACKET3_RELEASE_MEM__GCR_GL2_DISCARD (1 << 19) +#define PACKET3_RELEASE_MEM__GCR_GL2_INV (1 << 20) +#define PACKET3_RELEASE_MEM__GCR_GL2_WB (1 << 21) +#define PACKET3_RELEASE_MEM__GCR_SEQ(x) ((x) << 22) +#define PACKET3_RELEASE_MEM__GCR_GLV_WB (1 << 24) +#define PACKET3_RELEASE_MEM__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25) +#define PACKET3_RELEASE_MEM__TEMPORAL__RT 0 +#define PACKET3_RELEASE_MEM__TEMPORAL__NT 1 +#define PACKET3_RELEASE_MEM__TEMPORAL__HT 2 +#define PACKET3_RELEASE_MEM__TEMPORAL__LU 3 /* 0 - temporal__release_mem__rt * 1 - temporal__release_mem__nt * 2 - temporal__release_mem__ht * 3 - temporal__release_mem__lu */ -#define PACKET3_RELEASE_MEM_PQ_EXE_STATUS (1 << 28) -#define PACKET3_RELEASE_MEM_GCR_GLK_INV (1 << 30) +#define PACKET3_RELEASE_MEM__PQ_EXE_STATUS(x) ((((unsigned)(x)) & 0x1) << 28) +#define PACKET3_RELEASE_MEM__PQ_EXE_STATUS__DEFAULT 0 +#define PACKET3_RELEASE_MEM__PQ_EXE_STATUS__PHASE_UPDATE 1 +#define PACKET3_RELEASE_MEM__GCR_GLK_INV (1 << 30) -#define PACKET3_RELEASE_MEM_DST_SEL(x) ((x) << 16) - /* 0 - memory controller - * 1 - TC/L2 - * 2 - register - */ -#define PACKET3_RELEASE_MEM_MES_INTR_PIPE(x) ((x) << 20) -#define PACKET3_RELEASE_MEM_MES_ACTION_ID(x) ((x) << 22) -#define PACKET3_RELEASE_MEM_INT_SEL(x) ((x) << 24) - /* 0 - none - * 1 - interrupt only (DATA_SEL = 0) - * 2 - interrupt when data write is confirmed - */ -#define PACKET3_RELEASE_MEM_ADD_DOOREBLL_OFFSET(x) (1 << 28) -#define PACKET3_RELEASE_MEM_DATA_SEL(x) ((x) << 29) +#define PACKET3_RELEASE_MEM__DST_SEL(x) ((((unsigned)(x)) & 0x3) << 16) +#define PACKET3_RELEASE_MEM__DST_SEL__MEMORY_CONTROLLER 0 +#define PACKET3_RELEASE_MEM__DST_SEL__TC_L2 1 +#define PACKET3_RELEASE_MEM__DST_SEL__QUQUE_WRITE_POINTER_REGISTER 2 +#define PACKET3_RELEASE_MEM__DST_SEL__QUQUE_WRITE_POINTER_POLL_MASK_BIT 3 +#define PACKET3_RELEASE_MEM__MES_INTR_PIPE(x) ((((unsigned)(x)) & 0x3) << 20) +#define PACKET3_RELEASE_MEM__MES_ACTION_ID(x) ((((unsigned)(x)) & 0x3) << 22) +#define PACKET3_RELEASE_MEM__MES_ACTION_ID__NO_MES_NOTIFICATION 0 +#define PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_AND_FENCE 1 +#define PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_NO_FENCE_THEN_ADDRESS_PAYLOAD 2 +#define PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_AND_ADDRESS_PAYLOAD 3 +#define PACKET3_RELEASE_MEM__INT_SEL(x) ((((unsigned)(x)) & 0x7) << 24) +#define PACKET3_RELEASE_MEM__INT_SEL__NONE 0 +#define PACKET3_RELEASE_MEM__INT_SEL__SEND_INTERRUPT_ONLY 1 +#define PACKET3_RELEASE_MEM__INT_SEL__SEND_INTERRUPT_AFTER_WRITE_CONFIRM 2 +#define PACKET3_RELEASE_MEM__INT_SEL__SEND_DATA_AND_WRITE_CONFIRM 3 +#define PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID 4 +#define PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID_BASED_ON_32_BIT_COMPARE 5 +#define PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID_BASED_ON_64_BIT_COMPARE 6 +#define PACKET3_RELEASE_MEM__ADD_DOOREBLL_OFFSET(x) ((((unsigned)(x)) & 0x1) << 28) +#define PACKET3_RELEASE_MEM__DATA_SEL(x) ((((unsigned)(x)) & 0x7) << 29) +#define PACKET3_RELEASE_MEM__DATA_SEL__NONE 0 +#define PACKET3_RELEASE_MEM__DATA_SEL__SEND_32_BIT_LOW 1 +#define PACKET3_RELEASE_MEM__DATA_SEL__SEND_64_BIT_DATA 2 +#define PACKET3_RELEASE_MEM__DATA_SEL__SEND_GPU_CLOCK_COUNTER 3 +#define PACKET3_RELEASE_MEM__DATA_SEL__SEND_SYSTEM_CLOCK_COUNTER 4 /* 0 - discard * 1 - send low 32bit data * 2 - send 64bit data * 3 - send 64bit GPU counter value * 4 - send 64bit sys counter value */ +#define PACKET3_RELEASE_MEM__ADDRESS_LO_32B(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2) +#define PACKET3_RELEASE_MEM__ADDRESS_LO_64B(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3) +#define PACKET3_RELEASE_MEM__ADDRESS_HI(x) ((unsigned)(x)) +#define PACKET3_RELEASE_MEM__DATA_LO(x) ((unsigned)(x)) +#define PACKET3_RELEASE_MEM__CMP_DATA_LO(x) ((unsigned)(x)) +#define PACKET3_RELEASE_MEM__DATA_HI(x) ((unsigned)(x)) +#define PACKET3_RELEASE_MEM__CMP_DATA_HI(x) ((unsigned)(x)) +#define PACKET3_RELEASE_MEM__INT_CTXID(x) ((unsigned)(x)) #define PACKET3_PREAMBLE_CNTL 0x4A # define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE (2 << 28) @@ -279,58 +438,78 @@ # define PACKET3_DMA_DATA_CMD_DIS_WC (1 << 30) #define PACKET3_CONTEXT_REG_RMW 0x51 #define PACKET3_ACQUIRE_MEM 0x58 -/* 1. HEADER - * 2. COHER_CNTL [30:0] - * 2.1 ENGINE_SEL [31:31] - * 2. COHER_SIZE [31:0] - * 3. COHER_SIZE_HI [7:0] - * 4. COHER_BASE_LO [31:0] - * 5. COHER_BASE_HI [23:0] - * 7. POLL_INTERVAL [15:0] - * 8. GCR_CNTL [18:0] - */ -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(x) ((x) << 0) +/* 1. HEADER */ +#define PACKET3_ACQUIRE_MEM__COHER_SIZE(x) ((unsigned)(x)) +/* 3. COHER_SIZE [31:0] */ +#define PACKET3_ACQUIRE_MEM__COHER_SIZE_HI(x) ((((unsigned)(x)) & 0xFF) << 0) +/* 4. COHER_SIZE_HI [7:0] */ +#define PACKET3_ACQUIRE_MEM__COHER_BASE_LO(x) ((unsigned)(x)) +/* 5. COHER_BASE_LO [31:0] */ +#define PACKET3_ACQUIRE_MEM__COHER_BASE_HI(x) ((((unsigned)(x)) & 0xFFFFFF) << 0) +/* 6. COHER_BASE_HI [23:0] */ +#define PACKET3_ACQUIRE_MEM__POLL_INTERVAL(x) ((((unsigned)(x)) & 0xFFFF) << 0) +/* 7. POLL_INTERVAL [15:0] */ +#define PACKET3_ACQUIRE_MEM__GCR_CNTL(x) ((((unsigned)(x)) & 0x7FFFF) << 0) +/* 8. GCR_CNTL [18:0] */ +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV(x) ((x) << 0) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__NOP 0 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__ALL 1 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__RANGE 2 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__FIRST_LAST 3 /* * 0:NOP * 1:ALL * 2:RANGE * 3:FIRST_LAST */ -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_RANGE(x) ((x) << 2) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE(x) ((x) << 2) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__ALL 0 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__RANGE 2 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__FIRST_LAST 3 /* * 0:ALL * 1:reserved * 2:RANGE * 3:FIRST_LAST */ -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(x) ((x) << 4) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE(x) ((x) << 4) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__DEVICE 0 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__SYSTEM 1 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__FORCE_ALL 2 /* * 0:Device scope * 1:System scope * 2:Force INV/WB all * 3:Reserved */ -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_WB(x) ((x) << 6) -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(x) ((x) << 7) -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(x) ((x) << 8) -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_US(x) ((x) << 10) -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_RANGE(x) ((x) << 11) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_WB(x) ((x) << 6) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLK_INV(x) ((x) << 7) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_INV(x) ((x) << 8) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_US(x) ((x) << 10) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE(x) ((x) << 11) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__ALL 0 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__VOL 1 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__RANGE 2 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__FIRST_LAST 3 /* * 0:ALL * 1:VOL * 2:RANGE * 3:FIRST_LAST */ -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_DISCARD(x) ((x) << 13) -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(x) ((x) << 14) -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(x) ((x) << 15) -#define PACKET3_ACQUIRE_MEM_GCR_CNTL_SEQ(x) ((x) << 16) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_DISCARD(x) ((x) << 13) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_INV(x) ((x) << 14) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_WB(x) ((x) << 15) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ(x) ((x) << 16) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__PARALLET 0 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__FORWARD 1 +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__REVERSE 2 /* * 0: PARALLEL * 1: FORWARD * 2: REVERSE */ -#define PACKET3_ACQUIRE_MEM_GCR_RANGE_IS_PA (1 << 18) +#define PACKET3_ACQUIRE_MEM__GCR_CNTL__GCR_RANGE_IS_PA (1 << 18) #define PACKET3_GEN_PDEPTE 0x5B #define PACKET3_PRIME_UTCL2 0x5D #define PACKET3_LOAD_UCONFIG_REG 0x5E @@ -348,11 +527,19 @@ #define PACKET3_SET_SH_REG 0x76 #define PACKET3_SET_SH_REG_START 0x00002c00 #define PACKET3_SET_SH_REG_END 0x00003000 +#define PACKET3_SET_SH_REG__REG_OFFSET(x) ((((unsigned)(x)) & 0xFFFF) << 0) +#define PACKET3_SET_SH_REG__VMID_SHIFT(x) ((((unsigned)(x)) & 0x1F) << 23) +#define PACKET3_SET_SH_REG__INDEX(x) ((((unsigned)(x)) & 0xF) << 28) +#define PACKET3_SET_SH_REG__REG_DATA(x) (((unsigned)(x))) +#define PACKET3_SET_SH_REG__INDEX__DEFAULT 0 +#define PACKET3_SET_SH_REG__INDEX__INSERT_VMID 1 #define PACKET3_SET_SH_REG_OFFSET 0x77 #define PACKET3_SET_QUEUE_REG 0x78 #define PACKET3_SET_UCONFIG_REG 0x79 #define PACKET3_SET_UCONFIG_REG_START 0x0000c000 #define PACKET3_SET_UCONFIG_REG_END 0x0000c400 +#define PACKET3_SET_UCONFIG_REG__REG_OFFSET(x) ((((unsigned)(x)) & 0xFFFF) << 0) +#define PACKET3_SET_UCONFIG_REG__REG_DATA(x) (((unsigned)(x))) #define PACKET3_SET_UCONFIG_REG_INDEX 0x7A #define PACKET3_DISPATCH_DRAW_PREAMBLE 0x8C #define PACKET3_DISPATCH_DRAW 0x8D diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 2b691452775b..65b8497ad5f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -2819,6 +2819,7 @@ static void gfx_v7_0_mqd_init(struct amdgpu_device *adev, { u64 hqd_gpu_addr; u64 wb_gpu_addr; + u32 tmp; /* init the mqd struct */ memset(mqd, 0, sizeof(struct cik_mqd)); @@ -2923,7 +2924,11 @@ static void gfx_v7_0_mqd_init(struct amdgpu_device *adev, mqd->cp_hqd_atomic1_preop_lo = RREG32(mmCP_HQD_ATOMIC1_PREOP_LO); mqd->cp_hqd_atomic1_preop_hi = RREG32(mmCP_HQD_ATOMIC1_PREOP_HI); mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); - mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); + tmp = RREG32(mmCP_HQD_QUANTUM); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10); + mqd->cp_hqd_quantum = tmp; mqd->cp_hqd_pipe_priority = RREG32(mmCP_HQD_PIPE_PRIORITY); mqd->cp_hqd_queue_priority = RREG32(mmCP_HQD_QUEUE_PRIORITY); mqd->cp_hqd_iq_rptr = RREG32(mmCP_HQD_IQ_RPTR); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index a6b4c8f41dc1..70ba81e6b4d4 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -4532,7 +4532,11 @@ static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring) /* set static priority for a queue/ring */ gfx_v8_0_mqd_set_priority(ring, mqd); - mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); + tmp = RREG32(mmCP_HQD_QUANTUM); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10); + mqd->cp_hqd_quantum = tmp; /* map_queues packet doesn't need activate the queue, * so only kiq need set this field. @@ -6252,9 +6256,6 @@ static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring, static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, u64 seq, unsigned int flags) { - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 86c7c2a429b7..3370f542e990 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1183,7 +1183,7 @@ static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -3667,7 +3667,11 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) /* set static priority for a queue/ring */ gfx_v9_0_mqd_set_priority(ring, mqd); - mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); + tmp = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); + mqd->cp_hqd_quantum = tmp; /* map_queues packet doesn't need activate the queue, * so only kiq need set this field. @@ -4046,18 +4050,69 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block) !amdgpu_sriov_vf(adev)) gfx_v9_4_2_set_power_brake_sequence(adev); + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; + + return 0; + +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); return r; } +static void gfx_v9_0_deactivate_kcq_hqd(struct amdgpu_device *adev) +{ + amdgpu_gfx_rlc_enter_safe_mode(adev, 0); + for (int i = 0; i < adev->gfx.num_compute_rings; i++) { + u32 tmp; + struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; + + mutex_lock(&adev->srbm_mutex); + soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); + tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); + /* disable the queue if it's active */ + if (tmp & CP_HQD_ACTIVE__ACTIVE_MASK) { + int j; + + WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); + for (j = 0; j < adev->usec_timeout; j++) { + tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); + if (!(tmp & CP_HQD_ACTIVE__ACTIVE_MASK)) + break; + udelay(1); + } + if (j == AMDGPU_MAX_USEC_TIMEOUT) { + DRM_DEBUG("comp_%u_%u_%u dequeue request failed.\n", + ring->me, ring->pipe, ring->queue); + /* Manual disable if dequeue request times out */ + WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, 0); + } + WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0); + } + soc15_grbm_select(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); + } + amdgpu_gfx_rlc_exit_safe_mode(adev, 0); +} + static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) - amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); /* DF freeze and kcq disable will fail */ if (!amdgpu_ras_intr_triggered()) @@ -4075,6 +4130,10 @@ static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) return 0; } + if ((adev->flags & AMD_IS_APU) && amdgpu_in_reset(adev) && + amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_MODE2) + gfx_v9_0_deactivate_kcq_hqd(adev); + /* Use deinitialize sequence from CAIL when unbinding device from driver, * otherwise KIQ is hanging when binding back */ @@ -4858,18 +4917,6 @@ static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block) struct amdgpu_device *adev = ip_block->adev; int r; - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; - r = gfx_v9_0_ecc_late_init(ip_block); if (r) return r; @@ -5427,7 +5474,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5523,7 +5570,7 @@ static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5564,9 +5611,9 @@ static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index ad4d442e7345..2a36647b975a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -405,7 +405,7 @@ static void gfx_v9_4_3_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -1943,7 +1943,11 @@ static int gfx_v9_4_3_xcc_mqd_init(struct amdgpu_ring *ring, int xcc_id) /* set static priority for a queue/ring */ gfx_v9_4_3_mqd_set_priority(ring, mqd); - mqd->cp_hqd_quantum = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_QUANTUM); + tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_QUANTUM); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); + tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); + mqd->cp_hqd_quantum = tmp; /* map_queues packet doesn't need activate the queue, * so only kiq need set this field. @@ -2367,17 +2371,102 @@ static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block) if (r) return r; + r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); + if (r) + return r; + + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); + if (r) + goto err_priv_inst; + + r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); + if (r) + goto err_bad_op; + + return 0; + +err_bad_op: + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); +err_priv_inst: + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); return r; } +static int gfx_v9_4_3_perf_monitor_ptl_init(struct amdgpu_device *adev, bool enable) +{ + struct amdgpu_ptl *ptl = &adev->psp.ptl; + uint32_t ptl_state = enable ? 1 : 0; + uint32_t fmt1, fmt2; + int r; + + if (!adev->psp.funcs) + return -EOPNOTSUPP; + + if (!ptl->hw_supported) { + fmt1 = GFX_FTYPE_VECTOR; + fmt2 = GFX_FTYPE_F8; + } else { + fmt1 = ptl->fmt1; + fmt2 = ptl->fmt2; + } + + /* initialize PTL with default formats: GFX_FTYPE_VECTOR & GFX_FTYPE_F8 */ + r = amdgpu_ptl_perf_monitor_ctrl(adev, PSP_PTL_PERF_MON_SET, &ptl_state, + &fmt1, &fmt2); + if (r) + return r; + + ptl->hw_supported = true; + + atomic_set(&ptl->disable_ref, 0); + if (!enable && !amdgpu_in_reset(adev) && !adev->in_suspend) { + dev_dbg(adev->dev, + "PTL disabled (amdgpu.ptl=%d)\ + To enable, set amdgpu.ptl=1 via module param or kernel cmdline\n", + amdgpu_ptl); + set_bit(AMDGPU_PTL_DISABLE_SYSFS, ptl->disable_bitmap); + } + + return 0; +} + +static int gfx_v9_4_3_ptl_hw_init(struct amdgpu_device *adev) +{ + struct amdgpu_ptl *ptl = &adev->psp.ptl; + bool enable; + + switch (amdgpu_ptl) { + case 1: + enable = true; + break; + case 2: + /* Permanently disabled - cannot be re-enabled */ + enable = false; + ptl->permanently_disabled = true; + break; + case -1: + case 0: + default: + enable = false; + break; + } + + gfx_v9_4_3_perf_monitor_ptl_init(adev, enable ? 1 : 0); + + return 0; +} + static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; int i, num_xcc; - amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); - amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + if (adev->psp.ptl.hw_supported && !amdgpu_in_reset(adev)) + gfx_v9_4_3_perf_monitor_ptl_init(adev, false); + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); num_xcc = NUM_XCC(adev->gfx.xcc_mask); for (i = 0; i < num_xcc; i++) { @@ -2401,12 +2490,21 @@ static bool gfx_v9_4_3_is_idle(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; int i, num_xcc; + u32 gc_ip_version; num_xcc = NUM_XCC(adev->gfx.xcc_mask); + gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0); + for (i = 0; i < num_xcc; i++) { - if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS), - GRBM_STATUS, GUI_ACTIVE)) - return false; + if (gc_ip_version == IP_VERSION(9, 4, 4)) { + if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS), + GRBM_STATUS, SPI_BUSY)) + return false; + } else { + if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS), + GRBM_STATUS, GUI_ACTIVE)) + return false; + } } return true; } @@ -2531,24 +2629,13 @@ static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block) static int gfx_v9_4_3_late_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - int r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); - if (r) - return r; - - r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); - if (r) - return r; if (adev->gfx.ras && adev->gfx.ras->enable_watchdog_timer) adev->gfx.ras->enable_watchdog_timer(adev); + gfx_v9_4_3_ptl_hw_init(adev); + return 0; } @@ -2857,7 +2944,7 @@ static void gfx_v9_4_3_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -2891,9 +2978,9 @@ static void gfx_v9_4_3_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -2953,9 +3040,6 @@ static void gfx_v9_4_3_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c index f2fe6f5bc7f7..652eea6eae4a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c @@ -321,7 +321,7 @@ static void gfxhub_v11_5_0_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c index efcaca70c27a..6cbf837d50dd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c @@ -326,7 +326,7 @@ static void gfxhub_v12_0_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c index f089f70571aa..9e6a6e13dec0 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c @@ -318,7 +318,7 @@ static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c index 128115a2cb45..b3b1085c7cd3 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c @@ -323,7 +323,7 @@ static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index f5bdfea54afa..6be2000c8261 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -707,20 +707,16 @@ static int gmc_v10_0_mc_init(struct amdgpu_device *adev) adev->gmc.visible_vram_size = adev->gmc.aper_size; /* set the gart size */ - if (amdgpu_gart_size == -1) { - switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { - default: - adev->gmc.gart_size = 512ULL << 20; - break; - case IP_VERSION(10, 3, 1): /* DCE SG support */ - case IP_VERSION(10, 3, 3): /* DCE SG support */ - case IP_VERSION(10, 3, 6): /* DCE SG support */ - case IP_VERSION(10, 3, 7): /* DCE SG support */ - adev->gmc.gart_size = 1024ULL << 20; - break; - } - } else { - adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; + switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { + case IP_VERSION(10, 3, 1): /* DCE SG support */ + case IP_VERSION(10, 3, 3): /* DCE SG support */ + case IP_VERSION(10, 3, 6): /* DCE SG support */ + case IP_VERSION(10, 3, 7): /* DCE SG support */ + amdgpu_gmc_set_gart_size(adev, SZ_1G); + break; + default: + amdgpu_gmc_set_gart_size(adev, SZ_512M); + break; } gmc_v10_0_vram_gtt_location(adev, &adev->gmc); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index 807bd180b9d4..c40d9c467204 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -585,6 +585,7 @@ static void gmc_v11_0_set_mmhub_funcs(struct amdgpu_device *adev) case IP_VERSION(3, 3, 1): case IP_VERSION(3, 3, 2): case IP_VERSION(3, 4, 0): + case IP_VERSION(3, 4, 2): adev->mmhub.funcs = &mmhub_v3_3_funcs; break; default: @@ -604,6 +605,9 @@ static void gmc_v11_0_set_gfxhub_funcs(struct amdgpu_device *adev) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): adev->gfxhub.funcs = &gfxhub_v11_5_0_funcs; break; default: @@ -709,10 +713,7 @@ static int gmc_v11_0_mc_init(struct amdgpu_device *adev) adev->gmc.visible_vram_size = adev->gmc.real_vram_size; /* set the gart size */ - if (amdgpu_gart_size == -1) - adev->gmc.gart_size = 512ULL << 20; - else - adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; + amdgpu_gmc_set_gart_size(adev, SZ_512M); gmc_v11_0_vram_gtt_location(adev, &adev->gmc); @@ -781,6 +782,9 @@ static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): case IP_VERSION(11, 5, 4): + case IP_VERSION(11, 5, 6): + case IP_VERSION(11, 7, 0): + case IP_VERSION(11, 7, 1): set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask); set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask); /* diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index 8dc9c053897b..84c93364d220 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -729,8 +729,10 @@ static int gmc_v12_0_mc_init(struct amdgpu_device *adev) int r; if (adev->gmc.xgmi.connected_to_cpu) - adev->gmc.mc_vram_size = - adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; + /* On A+A, manage driver allocation range to the local + * node segment and prevent allocations on remote HBM. + */ + adev->gmc.mc_vram_size = adev->gmc.xgmi.node_segment_size; else adev->gmc.mc_vram_size = adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; @@ -763,10 +765,7 @@ static int gmc_v12_0_mc_init(struct amdgpu_device *adev) adev->gmc.visible_vram_size = adev->gmc.real_vram_size; /* set the gart size */ - if (amdgpu_gart_size == -1) { - adev->gmc.gart_size = 512ULL << 20; - } else - adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; + amdgpu_gmc_set_gart_size(adev, SZ_512M); gmc_v12_0_vram_gtt_location(adev, &adev->gmc); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c index 7ea7b9c30bca..855cd29cbffa 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c @@ -112,6 +112,8 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev, const char *hub_name; int ret, xcc_id = 0; uint32_t status = 0; + const char *die_name; + char die_name_buf[32]; u64 addr; node_id = entry->node_id; @@ -201,6 +203,17 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev, dev_err(adev->dev, " in page starting at address 0x%016llx from IH client %d (%s)\n", addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]); + if (adev->irq.ih_funcs && + adev->irq.ih_funcs->node_id_to_die_name) { + die_name = adev->irq.ih_funcs->node_id_to_die_name(adev, node_id, + die_name_buf, + sizeof(die_name_buf)); + if (die_name) + dev_err(adev->dev, + " cookie node_id %d fault from die %s\n", + node_id, die_name); + } + if (amdgpu_sriov_vf(adev)) return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c index 6aa581b1c148..a914dd8183b5 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c @@ -328,24 +328,18 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev) adev->gmc.visible_vram_size = adev->gmc.aper_size; /* set the gart size */ - if (amdgpu_gart_size == -1) { - switch (adev->asic_type) { - case CHIP_HAINAN: /* no MM engines */ - default: - adev->gmc.gart_size = 256ULL << 20; - break; - case CHIP_VERDE: /* UVD, VCE do not support GPUVM */ - case CHIP_TAHITI: /* UVD, VCE do not support GPUVM */ - case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */ - case CHIP_OLAND: /* UVD, VCE do not support GPUVM */ - adev->gmc.gart_size = 1024ULL << 20; - break; - } - } else { - adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; + switch (adev->asic_type) { + case CHIP_VERDE: /* UVD, VCE do not support GPUVM */ + case CHIP_TAHITI: /* UVD, VCE do not support GPUVM */ + case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */ + case CHIP_OLAND: /* UVD, VCE do not support GPUVM */ + amdgpu_gmc_set_gart_size(adev, SZ_1G); + break; + case CHIP_HAINAN: /* no MM engines */ + default: + amdgpu_gmc_set_gart_size(adev, SZ_256M); + break; } - - adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; gmc_v6_0_vram_gtt_location(adev, &adev->gmc); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 2b0362c4d9eb..98db62cc8718 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -394,27 +394,21 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev) adev->gmc.visible_vram_size = adev->gmc.aper_size; /* set the gart size */ - if (amdgpu_gart_size == -1) { - switch (adev->asic_type) { - case CHIP_TOPAZ: /* no MM engines */ - default: - adev->gmc.gart_size = 256ULL << 20; - break; + switch (adev->asic_type) { #ifdef CONFIG_DRM_AMDGPU_CIK - case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */ - case CHIP_HAWAII: /* UVD, VCE do not support GPUVM */ - case CHIP_KAVERI: /* UVD, VCE do not support GPUVM */ - case CHIP_KABINI: /* UVD, VCE do not support GPUVM */ - case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */ - adev->gmc.gart_size = 1024ULL << 20; - break; + case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */ + case CHIP_HAWAII: /* UVD, VCE do not support GPUVM */ + case CHIP_KAVERI: /* UVD, VCE do not support GPUVM */ + case CHIP_KABINI: /* UVD, VCE do not support GPUVM */ + case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */ + amdgpu_gmc_set_gart_size(adev, SZ_1G); + break; #endif - } - } else { - adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; + case CHIP_TOPAZ: /* no MM engines */ + default: + amdgpu_gmc_set_gart_size(adev, SZ_256M); + break; } - - adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; gmc_v7_0_vram_gtt_location(adev, &adev->gmc); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index fbccfcb3d7cf..c2a41fa3a396 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -585,27 +585,21 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev) adev->gmc.visible_vram_size = adev->gmc.aper_size; /* set the gart size */ - if (amdgpu_gart_size == -1) { - switch (adev->asic_type) { - case CHIP_POLARIS10: /* all engines support GPUVM */ - case CHIP_POLARIS11: /* all engines support GPUVM */ - case CHIP_POLARIS12: /* all engines support GPUVM */ - case CHIP_VEGAM: /* all engines support GPUVM */ - default: - adev->gmc.gart_size = 256ULL << 20; - break; - case CHIP_TONGA: /* UVD, VCE do not support GPUVM */ - case CHIP_FIJI: /* UVD, VCE do not support GPUVM */ - case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */ - case CHIP_STONEY: /* UVD does not support GPUVM, DCE SG support */ - adev->gmc.gart_size = 1024ULL << 20; - break; - } - } else { - adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; + switch (adev->asic_type) { + case CHIP_TONGA: /* UVD, VCE do not support GPUVM */ + case CHIP_FIJI: /* UVD, VCE do not support GPUVM */ + case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */ + case CHIP_STONEY: /* UVD does not support GPUVM, DCE SG support */ + amdgpu_gmc_set_gart_size(adev, SZ_1G); + break; + case CHIP_POLARIS10: /* all engines support GPUVM */ + case CHIP_POLARIS11: /* all engines support GPUVM */ + case CHIP_POLARIS12: /* all engines support GPUVM */ + case CHIP_VEGAM: /* all engines support GPUVM */ + default: + amdgpu_gmc_set_gart_size(adev, SZ_256M); + break; } - - adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; gmc_v8_0_vram_gtt_location(adev, &adev->gmc); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index c6dbe25f2bd9..8a5c44810ba1 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -1204,21 +1204,6 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev, { int local_node, nid; - /* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system - * memory can use more efficient MTYPEs. - */ - if (!(adev->flags & AMD_IS_APU) || - amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 3)) - return; - - /* Only direct-mapped memory allows us to determine the NUMA node from - * the DMA address. - */ - if (!adev->ram_is_direct_mapped) { - dev_dbg_ratelimited(adev->dev, "RAM is not direct mapped\n"); - return; - } - /* MTYPE_NC is the same default and can be overridden. * MTYPE_UC will be present if the memory is extended-coherent * and can also be overridden. @@ -1231,11 +1216,7 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev, return; } - /* FIXME: Only supported on native mode for now. For carve-out, the - * NUMA affinity of the GPU/VM needs to come from the PCI info because - * memory partitions are not associated with different NUMA nodes. - */ - if (adev->gmc.is_app_apu && vm->mem_id >= 0) { + if (vm->mem_id >= 0) { local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node; } else { dev_dbg_ratelimited(adev->dev, "Only native mode APU is supported.\n"); @@ -1344,6 +1325,20 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) { adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; + + /* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes, local system + * memory can use more efficient MTYPEs. + * + * APUs mapping system memory may need different MTYPEs on different + * NUMA nodes. + * + * Only direct-mapped memory allows us to determine the NUMA node from + * the DMA address. + */ + adev->gmc.override_pte = adev->gmc.is_app_apu && + num_possible_nodes() > 1 && + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) && + adev->ram_is_direct_mapped; } static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) @@ -1736,31 +1731,25 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev) adev->gmc.visible_vram_size = adev->gmc.aper_size; /* set the gart size */ - if (amdgpu_gart_size == -1) { - switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { - case IP_VERSION(9, 0, 1): /* all engines support GPUVM */ - case IP_VERSION(9, 2, 1): /* all engines support GPUVM */ - case IP_VERSION(9, 4, 0): - case IP_VERSION(9, 4, 1): - case IP_VERSION(9, 4, 2): - case IP_VERSION(9, 4, 3): - case IP_VERSION(9, 4, 4): - case IP_VERSION(9, 5, 0): - default: - adev->gmc.gart_size = 512ULL << 20; - break; - case IP_VERSION(9, 1, 0): /* DCE SG support */ - case IP_VERSION(9, 2, 2): /* DCE SG support */ - case IP_VERSION(9, 3, 0): - adev->gmc.gart_size = 1024ULL << 20; - break; - } - } else { - adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; + switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { + case IP_VERSION(9, 1, 0): /* DCE SG support */ + case IP_VERSION(9, 2, 2): /* DCE SG support */ + case IP_VERSION(9, 3, 0): + amdgpu_gmc_set_gart_size(adev, SZ_1G); + break; + case IP_VERSION(9, 0, 1): /* all engines support GPUVM */ + case IP_VERSION(9, 2, 1): /* all engines support GPUVM */ + case IP_VERSION(9, 4, 0): + case IP_VERSION(9, 4, 1): + case IP_VERSION(9, 4, 2): + case IP_VERSION(9, 4, 3): + case IP_VERSION(9, 4, 4): + case IP_VERSION(9, 5, 0): + default: + amdgpu_gmc_set_gart_size(adev, SZ_512M); + break; } - adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; - gmc_v9_0_vram_gtt_location(adev, &adev->gmc); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c b/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c index 95b3f4e55ec3..699c274d357e 100644 --- a/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c +++ b/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c @@ -790,7 +790,7 @@ static void ih_v6_1_set_interrupt_funcs(struct amdgpu_device *adev) const struct amdgpu_ip_block_version ih_v6_1_ip_block = { .type = AMD_IP_BLOCK_TYPE_IH, .major = 6, - .minor = 0, + .minor = 1, .rev = 0, .funcs = &ih_v6_1_ip_funcs, }; diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c index 1fbe904f4223..6de9e87e04e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c @@ -798,6 +798,43 @@ static void ih_v7_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 return; } +/* + * ih_v7_0_node_id_to_die_name - Decode IH cookie node_id to a die name string + * + * Currently, only applies to IH v7_1. For other IH versions returns NULL. + * + * IH v7_1 node_id encoding: + * node_id[N:3] = MID index + * node_id[2:0] = sub-slot: 0=MID, 1=AID, 2-5=AID.XCD, 6-7=RSV + */ +static const char *ih_v7_0_node_id_to_die_name(struct amdgpu_device *adev, + unsigned int node_id, + char *buf, size_t size) +{ + int mid_id, sub_slot; + + /* Node ID to die name decoding is only defined for IH v7_1 currenlty. */ + if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) != IP_VERSION(7, 1, 0)) + return NULL; + + mid_id = node_id >> 3; + sub_slot = node_id & 0x7; + + if (mid_id > 1) + return "UNKNOWN"; + + if (sub_slot == 0) + snprintf(buf, size, "MID%d", mid_id); + else if (sub_slot == 1) + snprintf(buf, size, "AID%d", mid_id); + else if (sub_slot <= 5) + snprintf(buf, size, "AID%d.XCD%d", mid_id, sub_slot - 2); + else + snprintf(buf, size, "RSV"); + + return buf; +} + static const struct amd_ip_funcs ih_v7_0_ip_funcs = { .name = "ih_v7_0", .early_init = ih_v7_0_early_init, @@ -819,7 +856,8 @@ static const struct amdgpu_ih_funcs ih_v7_0_funcs = { .get_wptr = ih_v7_0_get_wptr, .decode_iv = amdgpu_ih_decode_iv_helper, .decode_iv_ts = amdgpu_ih_decode_iv_ts_helper, - .set_rptr = ih_v7_0_set_rptr + .set_rptr = ih_v7_0_set_rptr, + .node_id_to_die_name = ih_v7_0_node_id_to_die_name, }; static void ih_v7_0_set_interrupt_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c b/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c index 46d25d55ebbe..05b164f38c97 100644 --- a/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c @@ -42,6 +42,9 @@ MODULE_FIRMWARE("amdgpu/gc_11_5_1_imu.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_2_imu.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_3_imu.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_4_imu.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_5_6_imu.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_imu.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_imu.bin"); static int imu_v11_0_init_microcode(struct amdgpu_device *adev) { diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c index 539785a51f69..fe20cf2b9454 100644 --- a/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c @@ -140,27 +140,17 @@ static int imu_v12_1_switch_compute_partition(struct amdgpu_device *adev, int ret; if (adev->psp.funcs) { - /*TODO: revisit asp interface once it's avaialble */ - ret = psp_spatial_partition(&adev->psp, - NUM_XCC(adev->gfx.xcc_mask) / - num_xccs_per_xcp); + ret = psp_spatial_partition(&adev->psp, compute_partition_mode); if (ret) return ret; } adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp; - return 0; } -static void imu_v12_1_init_mcm_addr_lut(struct amdgpu_device *adev) -{ - /* todo: fill in when interface is ready */ -} - const struct amdgpu_imu_funcs gfx_v12_1_imu_funcs = { .init_microcode = imu_v12_1_init_microcode, .load_microcode = imu_v12_1_load_microcode, .switch_compute_partition = imu_v12_1_switch_compute_partition, - .init_mcm_addr_lut = imu_v12_1_init_mcm_addr_lut, }; diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c index 13a6e24c624a..5208312e7017 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c @@ -149,7 +149,7 @@ static int jpeg_v2_5_sw_init(struct amdgpu_ip_block *ip_block) else ring->vm_hub = AMDGPU_MMHUB0(0); ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + 8 * i; - sprintf(ring->name, "jpeg_dec_%d", i); + snprintf(ring->name, sizeof(ring->name), "jpeg_dec_%d", i); r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst[i].irq, 0, AMDGPU_RING_PRIO_DEFAULT, NULL); if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c index 0c746580de11..d8204fbc198d 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c @@ -1010,7 +1010,7 @@ void jpeg_v4_0_3_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count) static bool jpeg_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - bool ret = false; + bool ret = true; int i, j; for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c index 250316704dfa..ae3afc7ab326 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c @@ -657,7 +657,7 @@ static void jpeg_v5_0_1_dec_ring_set_wptr(struct amdgpu_ring *ring) static bool jpeg_v5_0_1_is_idle(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - bool ret = false; + bool ret = true; int i, j; for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c index 4cbd46f53e85..16625c31bfd3 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -251,7 +251,7 @@ static int mes_userq_detect_and_reset(struct amdgpu_device *adev, if (found_hung_queue) { /* Resume scheduling after hang recovery */ - r = amdgpu_mes_resume(adev); + r = amdgpu_mes_resume(adev, input.xcc_id); } return r; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index a926a330700e..1b071a3de173 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -58,6 +58,12 @@ MODULE_FIRMWARE("amdgpu/gc_11_5_3_mes_2.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_3_mes1.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_4_mes_2.bin"); MODULE_FIRMWARE("amdgpu/gc_11_5_4_mes1.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_5_6_mes_2.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_5_6_mes1.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_mes_2.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_0_mes1.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_mes_2.bin"); +MODULE_FIRMWARE("amdgpu/gc_11_7_1_mes1.bin"); static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block); static int mes_v11_0_hw_fini(struct amdgpu_ip_block *ip_block); @@ -557,6 +563,7 @@ static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes, mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; + mes_suspend_gang_pkt.doorbell_offset = input->doorbell_offset; return mes_v11_0_submit_pkt_and_poll_completion(mes, &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt), @@ -576,6 +583,7 @@ static int mes_v11_0_resume_gang(struct amdgpu_mes *mes, mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_resume_gang_pkt.doorbell_offset = input->doorbell_offset; return mes_v11_0_submit_pkt_and_poll_completion(mes, &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt), @@ -1686,6 +1694,7 @@ static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block) if (r) goto failure; + amdgpu_mes_validate_fw_version(adev); out: /* * Disable KIQ ring usage from the driver once MES is enabled. diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c index 023c7345ea54..b6cbc25e1ab4 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c @@ -592,6 +592,7 @@ static int mes_v12_0_suspend_gang(struct amdgpu_mes *mes, mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; + mes_suspend_gang_pkt.doorbell_offset = input->doorbell_offset; return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE, &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt), @@ -611,6 +612,7 @@ static int mes_v12_0_resume_gang(struct amdgpu_mes *mes, mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_resume_gang_pkt.doorbell_offset = input->doorbell_offset; return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE, &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt), @@ -1871,6 +1873,7 @@ static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block) if (r) goto failure; + amdgpu_mes_validate_fw_version(adev); out: /* * Disable KIQ ring usage from the driver once MES is enabled. diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c index cec801278126..e13535d94c51 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c @@ -44,8 +44,11 @@ static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block); static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id); static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id); static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id); +static int mes_v12_1_setup_coop_mode(struct amdgpu_device *adev, int xcc_id); #define MES_EOP_SIZE 2048 +#define MES12_HUNG_DB_OFFSET_ARRAY_SIZE 8 /* [0:3] = db offset [4:7] hqd info */ +#define MES12_HUNG_HQD_INFO_OFFSET 4 #define regCP_HQD_IB_CONTROL_MES_12_1_DEFAULT 0x100000 #define XCC_MID_MASK 0x41000000 @@ -229,7 +232,7 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes, xcc_id, pipe, x_pkt->header.opcode); r = amdgpu_fence_wait_polling(ring, seq, timeout); - if (r < 1 || !*status_ptr) { + if (r < 1 || !lower_32_bits(*status_ptr)) { if (misc_op_str) dev_err(adev->dev, "MES(%d, %d) failed to respond to msg=%s (%s)\n", @@ -468,13 +471,46 @@ static int mes_v12_1_unmap_legacy_queue(struct amdgpu_mes *mes, static int mes_v12_1_suspend_gang(struct amdgpu_mes *mes, struct mes_suspend_gang_input *input) { - return 0; + union MESAPI__SUSPEND mes_suspend_gang_pkt; + + memset(&mes_suspend_gang_pkt, 0, sizeof(mes_suspend_gang_pkt)); + + mes_suspend_gang_pkt.header.type = MES_API_TYPE_SCHEDULER; + mes_suspend_gang_pkt.header.opcode = MES_SCH_API_SUSPEND; + mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; + + mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs; + mes_suspend_gang_pkt.suspend_all_sdma_gangs = input->suspend_all_sdma_gangs; + mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; + mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; + mes_suspend_gang_pkt.doorbell_offset = input->doorbell_offset; + + /* Suspend gang is handled by master MES */ + return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE, + &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt), + offsetof(union MESAPI__SUSPEND, api_status)); } static int mes_v12_1_resume_gang(struct amdgpu_mes *mes, struct mes_resume_gang_input *input) { - return 0; + union MESAPI__RESUME mes_resume_gang_pkt; + + memset(&mes_resume_gang_pkt, 0, sizeof(mes_resume_gang_pkt)); + + mes_resume_gang_pkt.header.type = MES_API_TYPE_SCHEDULER; + mes_resume_gang_pkt.header.opcode = MES_SCH_API_RESUME; + mes_resume_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; + + mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs; + mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr; + mes_resume_gang_pkt.doorbell_offset = input->doorbell_offset; + + /* Resume gang is handled by master MES */ + return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE, + &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt), + offsetof(union MESAPI__RESUME, api_status)); } static int mes_v12_1_query_sched_status(struct amdgpu_mes *mes, @@ -621,11 +657,15 @@ static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes, mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 100; - if (mes->enable_coop_mode && pipe == AMDGPU_MES_SCHED_PIPE) { + /* From version 0x74 above, pipe1 support use shared command buffer + to distribute some tasks on individual XCCs*/ + if (mes->enable_coop_mode && + ((pipe == AMDGPU_MES_SCHED_PIPE) || + ((mes->kiq_version & AMDGPU_MES_VERSION_MASK) >= 0x74))) { master_xcc_id = mes->master_xcc_ids[inst]; mes_set_hw_res_1_pkt.mes_coop_mode = 1; mes_set_hw_res_1_pkt.coop_sch_shared_mc_addr = - mes->shared_cmd_buf_gpu_addr[master_xcc_id]; + mes->shared_cmd_buf_gpu_addr[master_xcc_id + pipe]; } return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe, @@ -633,21 +673,10 @@ static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes, offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status)); } -static void mes_v12_1_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt) -{ - /* - * GFX V12 has only one GFX pipe, but 8 queues in it. - * GFX pipe 0 queue 0 is being used by Kernel queue. - * Set GFX pipe 0 queue 1-7 for MES scheduling - * mask = 1111 1110b - */ - pkt->gfx_hqd_mask[0] = 0xFE; -} - static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes, int pipe, int xcc_id) { - int i; + int i, status; struct amdgpu_device *adev = mes->adev; union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt; @@ -667,7 +696,9 @@ static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes, mes_set_hw_res_pkt.compute_hqd_mask[i] = mes->compute_hqd_mask[i]; - mes_v12_1_set_gfx_hqd_mask(&mes_set_hw_res_pkt); + for (i = 0; i < MAX_GFX_PIPES; i++) + mes_set_hw_res_pkt.gfx_hqd_mask[i] = + mes->gfx_hqd_mask[i]; for (i = 0; i < MAX_SDMA_PIPES; i++) mes_set_hw_res_pkt.sdma_hqd_mask[i] = @@ -715,9 +746,23 @@ static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes, if (adev->enforce_isolation[0] == AMDGPU_ENFORCE_ISOLATION_ENABLE) mes_set_hw_res_pkt.limit_single_process = 1; - return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe, + status = mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe, &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt), offsetof(union MESAPI_SET_HW_RESOURCES, api_status)); + + /* get MES scheduler versions */ + mutex_lock(&adev->srbm_mutex); + soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id)); + + if (pipe == AMDGPU_MES_SCHED_PIPE) + adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO); + else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq) + adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO); + + soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); + mutex_unlock(&adev->srbm_mutex); + + return status; } static void mes_v12_1_init_aggregated_doorbell(struct amdgpu_mes *mes, @@ -837,6 +882,33 @@ static int mes_v12_1_reset_legacy_queue(struct amdgpu_mes *mes, } #endif +static int mes_v12_1_detect_and_reset_hung_queues(struct amdgpu_mes *mes, + struct mes_detect_and_reset_queue_input *input) +{ + union MESAPI__RESET mes_reset_queue_pkt; + + memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt)); + + mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER; + mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET; + mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; + + mes_reset_queue_pkt.queue_type = + convert_to_mes_queue_type(input->queue_type); + mes_reset_queue_pkt.doorbell_offset_addr = + mes->hung_queue_db_array_gpu_addr[0]; + + if (input->detect_only) + mes_reset_queue_pkt.hang_detect_only = 1; + else + mes_reset_queue_pkt.hang_detect_then_reset = 1; + + return mes_v12_1_submit_pkt_and_poll_completion(mes, + input->xcc_id, AMDGPU_MES_SCHED_PIPE, + &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt), + offsetof(union MESAPI__RESET, api_status)); +} + static int mes_v12_inv_tlb_convert_hub_id(uint8_t id) { /* @@ -894,6 +966,7 @@ static const struct amdgpu_mes_funcs mes_v12_1_funcs = { .resume_gang = mes_v12_1_resume_gang, .misc_op = mes_v12_1_misc_op, .reset_hw_queue = mes_v12_1_reset_hw_queue, + .detect_and_reset_hung_queues = mes_v12_1_detect_and_reset_hung_queues, .invalidate_tlbs_pasid = mes_v12_1_inv_tlbs_pasid, }; @@ -1157,9 +1230,6 @@ static int mes_v12_1_allocate_shared_cmd_buf(struct amdgpu_device *adev, { int r, inst = MES_PIPE_INST(xcc_id, pipe); - if (pipe == AMDGPU_MES_KIQ_PIPE) - return 0; - r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, &adev->mes.shared_cmd_buf_obj[inst], @@ -1405,18 +1475,6 @@ static int mes_v12_1_queue_init(struct amdgpu_device *adev, mes_v12_1_queue_init_register(ring, xcc_id); } - /* get MES scheduler/KIQ versions */ - mutex_lock(&adev->srbm_mutex); - soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id)); - - if (pipe == AMDGPU_MES_SCHED_PIPE) - adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO); - else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq) - adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO); - - soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); - mutex_unlock(&adev->srbm_mutex); - return 0; } @@ -1723,6 +1781,10 @@ static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id) goto failure; if (adev->enable_uni_mes) { + r = mes_v12_1_setup_coop_mode(adev, xcc_id); + if (r) + goto failure; + r = mes_v12_1_set_hw_resources(&adev->mes, AMDGPU_MES_KIQ_PIPE, xcc_id); if (r) @@ -1845,9 +1907,6 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id) goto failure; if (adev->enable_uni_mes) { - r = mes_v12_1_setup_coop_mode(adev, xcc_id); - if (r) - goto failure; mes_v12_1_set_hw_resources_1(&adev->mes, AMDGPU_MES_SCHED_PIPE, xcc_id); } @@ -1860,6 +1919,7 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id) goto failure; } + amdgpu_mes_validate_fw_version(adev); out: /* * Disable KIQ ring usage from the driver once MES is enabled. @@ -1897,24 +1957,12 @@ static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block) static int mes_v12_1_suspend(struct amdgpu_ip_block *ip_block) { - int r; - - r = amdgpu_mes_suspend(ip_block->adev); - if (r) - return r; - return mes_v12_1_hw_fini(ip_block); } static int mes_v12_1_resume(struct amdgpu_ip_block *ip_block) { - int r; - - r = mes_v12_1_hw_init(ip_block); - if (r) - return r; - - return amdgpu_mes_resume(ip_block->adev); + return mes_v12_1_hw_init(ip_block); } static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block) @@ -1922,6 +1970,9 @@ static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block) struct amdgpu_device *adev = ip_block->adev; int pipe, r; + adev->mes.hung_queue_db_array_size = MES12_HUNG_DB_OFFSET_ARRAY_SIZE; + adev->mes.hung_queue_hqd_info_offset = MES12_HUNG_HQD_INFO_OFFSET; + for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) { r = amdgpu_mes_init_microcode(adev, pipe); if (r) @@ -2016,7 +2067,7 @@ static int mes_v12_1_map_test_bo(struct amdgpu_device *adev, error: amdgpu_sync_free(&sync); - return 0; + return r; } static int mes_v12_1_test_ring(struct amdgpu_device *adev, int xcc_id, diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c index 3d82cfa0f1b5..ab56dd15b3f5 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c @@ -340,7 +340,7 @@ static void mmhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c index a1b0b7b39a42..6522a89379b7 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c @@ -333,7 +333,7 @@ static void mmhub_v3_0_1_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c index 34e8dbd47c0f..23cf95783264 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_2.c @@ -339,7 +339,7 @@ static void mmhub_v3_0_2_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c index cfce7e1297d4..c3729ab9faa1 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c @@ -451,7 +451,7 @@ static void mmhub_v3_3_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, @@ -636,6 +636,7 @@ static void mmhub_v3_3_init_client_info(struct amdgpu_device *adev) ARRAY_SIZE(mmhub_client_ids_v3_3_1)); break; case IP_VERSION(3, 4, 0): + case IP_VERSION(3, 4, 2): amdgpu_mmhub_init_client_info(&adev->mmhub, mmhub_client_ids_v3_4, ARRAY_SIZE(mmhub_client_ids_v3_4)); diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c index bef75c4c48d3..c9fb48992a2d 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_1_0.c @@ -334,7 +334,7 @@ static void mmhub_v4_1_0_setup_vmid_config(struct amdgpu_device *adev) /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c index 29f7ed466858..49b7f16a941f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c @@ -518,7 +518,7 @@ static void mmhub_v4_2_0_mid_setup_vmid_config(struct amdgpu_device *adev, /* Send no-retry XNACK on fault to suppress VM fault storm. */ tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, - !amdgpu_noretry); + !adev->gmc.noretry); WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_CNTL, i * hub->ctx_distance, tmp); WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c index e1d63bed84bf..c3293e5a658c 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c @@ -308,7 +308,7 @@ void xgpu_vi_init_golden_registers(struct amdgpu_device *adev) xgpu_tonga_golden_common_all)); break; default: - BUG_ON("Doesn't support chip type.\n"); + dev_err(adev->dev, "Doesn't support chip type %d\n", adev->asic_type); break; } } diff --git a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c index b6f832c53860..c78f0598637f 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c @@ -57,12 +57,50 @@ #define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10 0x0021 #define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10_BASE_IDX 2 +#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_nbio_7_11_5 0x8e13 +#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_nbio_7_11_5_BASE_IDX 5 +#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_nbio_7_11_5 0x8e14 +#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_nbio_7_11_5_BASE_IDX 5 +#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL_nbio_7_11_5 0x8e15 +#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL_nbio_7_11_5_BASE_IDX 5 + +#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL_nbio_7_11_5 0x8e4d +#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL_nbio_7_11_5_BASE_IDX 5 +#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL_nbio_7_11_5 0x8e4e +#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL_nbio_7_11_5_BASE_IDX 5 + +#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbio_7_11_5 0xd000 +#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbio_7_11_5_BASE_IDX 5 + +#define regBIF_BX0_BIF_FB_EN_nbio_7_11_5 0x8e20 +#define regBIF_BX0_BIF_FB_EN_nbio_7_11_5_BASE_IDX 5 + +#define regBIF_BX0_INTERRUPT_CNTL_nbio_7_11_5 0x8e11 +#define regBIF_BX0_INTERRUPT_CNTL_nbio_7_11_5_BASE_IDX 5 +#define regBIF_BX0_INTERRUPT_CNTL2_nbio_7_11_5 0x8e12 +#define regBIF_BX0_INTERRUPT_CNTL2_nbio_7_11_5_BASE_IDX 5 + +#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ_nbio_7_11_5 0x8e26 +#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ_nbio_7_11_5_BASE_IDX 5 +#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE_nbio_7_11_5 0x8e27 +#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE_nbio_7_11_5_BASE_IDX 5 + +#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL_nbio_7_11_5 0x8e17 +#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL_nbio_7_11_5_BASE_IDX 5 + static void nbif_v6_3_1_remap_hdp_registers(struct amdgpu_device *adev) { - WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL, - adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); - WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL, - adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) { + WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL_nbio_7_11_5, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); + WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL_nbio_7_11_5, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); + } else { + WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); + WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); + } } static u32 nbif_v6_3_1_get_rev_id(struct amdgpu_device *adev) @@ -71,6 +109,8 @@ static u32 nbif_v6_3_1_get_rev_id(struct amdgpu_device *adev) if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10); + else if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) + tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbio_7_11_5); else tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0); @@ -82,12 +122,21 @@ static u32 nbif_v6_3_1_get_rev_id(struct amdgpu_device *adev) static void nbif_v6_3_1_mc_access_enable(struct amdgpu_device *adev, bool enable) { - if (enable) - WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, - BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK | - BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK); - else - WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0); + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) { + if (enable) + WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN_nbio_7_11_5, + BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK | + BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK); + else + WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN_nbio_7_11_5, 0); + } else { + if (enable) + WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, + BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK | + BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK); + else + WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0); + } } static u32 nbif_v6_3_1_get_memsize(struct amdgpu_device *adev) @@ -100,8 +149,14 @@ static void nbif_v6_3_1_sdma_doorbell_range(struct amdgpu_device *adev, int doorbell_index, int doorbell_size) { + u32 doorbell_range; if (instance == 0) { - u32 doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL); + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) + doorbell_range = RREG32_SOC15(NBIO, 0, + regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_nbif_4_10); + else + doorbell_range = RREG32_SOC15(NBIO, 0, + regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL); if (use_doorbell) { doorbell_range = REG_SET_FIELD(doorbell_range, @@ -130,11 +185,10 @@ static void nbif_v6_3_1_sdma_doorbell_range(struct amdgpu_device *adev, S2A_DOORBELL_PORT2_RANGE_SIZE, 0); - if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) { + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_nbif_4_10, doorbell_range); - } else { + else WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, doorbell_range); - } } } @@ -144,10 +198,13 @@ static void nbif_v6_3_1_vcn_doorbell_range(struct amdgpu_device *adev, { u32 doorbell_range; - if (instance) + if (instance) { + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) + return; doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL); - else + } else { doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL); + } if (use_doorbell) { doorbell_range = REG_SET_FIELD(doorbell_range, @@ -176,22 +233,67 @@ static void nbif_v6_3_1_vcn_doorbell_range(struct amdgpu_device *adev, S2A_DOORBELL_PORT4_RANGE_SIZE, 0); - if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) { - if (instance) - WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_nbif_4_10, doorbell_range); - else - WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10, doorbell_range); - } else { + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) + WREG32_SOC15(NBIO, 0, + regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10, + doorbell_range); + else if (instance) WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range); else WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL, doorbell_range); +} + +static void nbif_v6_3_1_vpe_doorbell_range(struct amdgpu_device *adev, + int instance, bool use_doorbell, + int doorbell_index, + int doorbell_size) +{ + if (instance) + return; + + u32 doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL); + + if (use_doorbell) { + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, + S2A_DOORBELL_PORT5_ENABLE, + 0x1); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, + S2A_DOORBELL_PORT5_AWID, + 0xf); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, + S2A_DOORBELL_PORT5_RANGE_OFFSET, + doorbell_index); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, + S2A_DOORBELL_PORT5_RANGE_SIZE, + doorbell_size); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, + S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE, + 0xf); + } else { + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, + S2A_DOORBELL_PORT5_RANGE_SIZE, + 0); + } + + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4) || + amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) { + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_nbif_4_10, doorbell_range); + } else { + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range); } + } static void nbif_v6_3_1_gc_doorbell_init(struct amdgpu_device *adev) { - if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) { + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) { WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL_nbif_4_10, 0x30000007); WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL_nbif_4_10, 0x3000000d); } else { @@ -233,7 +335,13 @@ nbif_v6_3_1_enable_doorbell_selfring_aperture(struct amdgpu_device *adev, static void nbif_v6_3_1_ih_doorbell_range(struct amdgpu_device *adev, bool use_doorbell, int doorbell_index) { - u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL); + u32 ih_doorbell_range; + + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) + ih_doorbell_range = RREG32_SOC15(NBIO, 0, + regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_nbif_4_10); + else + ih_doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL); if (use_doorbell) { ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, @@ -262,11 +370,11 @@ static void nbif_v6_3_1_ih_doorbell_range(struct amdgpu_device *adev, S2A_DOORBELL_PORT1_RANGE_SIZE, 0); - if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) { - WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_nbif_4_10, ih_doorbell_range); - } else { + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_nbif_4_10, + ih_doorbell_range); + else WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range); - } } static void nbif_v6_3_1_ih_control(struct amdgpu_device *adev) @@ -274,9 +382,13 @@ static void nbif_v6_3_1_ih_control(struct amdgpu_device *adev) u32 interrupt_cntl; /* setup interrupt control */ - WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8); + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) + WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2_nbio_7_11_5, + adev->dummy_page_addr >> 8); + else + WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8); - interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL); + interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL_nbio_7_11_5); /* * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN @@ -288,7 +400,10 @@ static void nbif_v6_3_1_ih_control(struct amdgpu_device *adev) interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0); - WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl); + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) + WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL_nbio_7_11_5, interrupt_cntl); + else + WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl); } static void @@ -311,27 +426,31 @@ nbif_v6_3_1_get_clockgating_state(struct amdgpu_device *adev, static u32 nbif_v6_3_1_get_hdp_flush_req_offset(struct amdgpu_device *adev) { - return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ); + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) + return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ_nbio_7_11_5); + else + return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ); } static u32 nbif_v6_3_1_get_hdp_flush_done_offset(struct amdgpu_device *adev) { - return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE); + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) + return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE_nbio_7_11_5); + else + return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE); } static u32 nbif_v6_3_1_get_pcie_index_offset(struct amdgpu_device *adev) { - if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) { + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX); - } - else { + else return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_INDEX); - } } static u32 nbif_v6_3_1_get_pcie_data_offset(struct amdgpu_device *adev) { - if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA); else return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_DATA); @@ -501,8 +620,12 @@ static void nbif_v6_3_1_set_reg_remap(struct amdgpu_device *adev) adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET; adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET; } else { - adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0, - regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2; + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) + adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0, + regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL_nbio_7_11_5) << 2; + else + adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0, + regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2; adev->rmmio_remap.bus_addr = 0; } } @@ -517,6 +640,7 @@ const struct amdgpu_nbio_funcs nbif_v6_3_1_funcs = { .get_memsize = nbif_v6_3_1_get_memsize, .sdma_doorbell_range = nbif_v6_3_1_sdma_doorbell_range, .vcn_doorbell_range = nbif_v6_3_1_vcn_doorbell_range, + .vpe_doorbell_range = nbif_v6_3_1_vpe_doorbell_range, .gc_doorbell_init = nbif_v6_3_1_gc_doorbell_init, .enable_doorbell_aperture = nbif_v6_3_1_enable_doorbell_aperture, .enable_doorbell_selfring_aperture = nbif_v6_3_1_enable_doorbell_selfring_aperture, diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c new file mode 100644 index 000000000000..5e8f466f23ad --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c @@ -0,0 +1,369 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "nbio/nbio_6_3_2_offset.h" +#include "nbio/nbio_6_3_2_sh_mask.h" + +#include "amdgpu.h" +#include "nbio_v6_3_2.h" + +static u32 nbio_v6_3_2_get_pcie_index_offset(struct amdgpu_device *adev) +{ + return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2); +} + +static u32 nbio_v6_3_2_get_pcie_data_offset(struct amdgpu_device *adev) +{ + return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA2); +} + +static u32 nbio_v6_3_2_get_pcie_index_hi_offset(struct amdgpu_device *adev) +{ + return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2_HI); +} + +static u32 nbio_v6_3_2_get_rev_id(struct amdgpu_device *adev) +{ + u32 tmp; + + /* TODO: RCC_STRAP0_RCC_DEV0_EPF0_STRAP0 is not accessible from + * guest side. It requires bootloader to update specific fields + * in ip discovery table to identify soc revision id. + * Return 0 when the function is called from guest side until + * bootloader change is available. + */ + if (amdgpu_sriov_vf(adev)) + return 0; + + tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0); + tmp = REG_GET_FIELD(tmp, RCC_STRAP0_RCC_DEV0_EPF0_STRAP0, + STRAP_ATI_REV_ID_DEV0_F0); + + return tmp; +} + +static void nbio_v6_3_2_mc_access_enable(struct amdgpu_device *adev, + bool enable) +{ + if (enable) + WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, + BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK | BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK); + else + WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0); +} + +static void nbio_v6_3_2_init_registers(struct amdgpu_device *adev) +{ + WREG32_SOC15(NBIO, 0, regXCD_DOORBELL_FENCE_1, + (0xff & ~(adev->gfx.xcc_mask)) << + XCD_DOORBELL_FENCE_1__XCD_0_DOORBELL_DISABLE__SHIFT); +} + +static u32 nbio_v6_3_2_get_memsize(struct amdgpu_device *adev) +{ + return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE); +} + +static void nbio_v6_3_2_enable_doorbell_aperture(struct amdgpu_device *adev, + bool enable) +{ + /* Enable to allow doorbell pass thru on pre-silicon bare-metal */ + WREG32_SOC15(NBIO, 0, regGDC0_DOORBELL_ACCESS_EN_PF, 0xfffff); + WREG32_FIELD15_PREREG(NBIO, 0, RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN, + BIF_DOORBELL_APER_EN, enable ? 1 : 0); +} + +static void nbio_v6_3_2_enable_doorbell_selfring_aperture(struct amdgpu_device *adev, + bool enable) +{ + u32 tmp = 0; + + if (enable) { + tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, + DOORBELL_SELFRING_GPA_APER_EN, 1) | + REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, + DOORBELL_SELFRING_GPA_APER_MODE, 1) | + REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, + DOORBELL_SELFRING_GPA_APER_SIZE, 0); + + WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW, + lower_32_bits(adev->doorbell.base)); + WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH, + upper_32_bits(adev->doorbell.base)); + } + + WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, tmp); +} + +static void nbio_v6_3_2_enable_doorbell_interrupt(struct amdgpu_device *adev, + bool enable) +{ + WREG32_FIELD15_PREREG(NBIO, 0, BIF_BX0_BIF_DOORBELL_INT_CNTL, + DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1); +} + +static void nbio_v6_3_2_ih_control(struct amdgpu_device *adev) +{ + u32 interrupt_cntl; + + /* setup interrupt control */ + WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8); + + interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL); + /* + * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi + * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN + */ + interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL, + IH_DUMMY_RD_OVERRIDE, 0); + + /* BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */ + interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL, + IH_REQ_NONSNOOP_EN, 0); + + WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl); +} + +static void nbio_v6_3_2_ih_doorbell_range(struct amdgpu_device *adev, + bool use_doorbell, int doorbell_index) +{ + u32 ih_doorbell_range = 0; + u32 ih_doorbell_range1 = 0; + + if (use_doorbell) { + ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, + S2A_DOORBELL_PORT1_ENABLE, + 0x1); + ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, + S2A_DOORBELL_PORT1_AWID, + 0x0); + ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, + S2A_DOORBELL_PORT1_RANGE_OFFSET, + doorbell_index); + ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, + S2A_DOORBELL_PORT1_RANGE_SIZE, + 8); + ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, + S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, + 0x0); + ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1, + S2A_DOORBELL_PORT1_TARGET_PORT_TYPE, + 0x3); + ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1, + S2A_DOORBELL_PORT1_TARGET_DIEID, + 0x0); + ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1, + S2A_DOORBELL_PORT1_TARGET_PORT_ID, + 0x0); + } + + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range); + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1, ih_doorbell_range1); +} + +static void nbio_v6_3_2_gc_doorbell_init(struct amdgpu_device *adev) +{ + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL, 0x30000007); + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1, 0x3); +} + +static void nbio_v6_3_2_sdma_doorbell_range(struct amdgpu_device *adev, + int instance, bool use_doorbell, + int doorbell_index, + int doorbell_size) +{ + if (instance == 0) { + u32 doorbell_range = 0; + u32 doorbell_range1 = 0; + + if (use_doorbell) { + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, + S2A_DOORBELL_PORT6_ENABLE, + 0x1); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, + S2A_DOORBELL_PORT6_AWID, + 0xe); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, + S2A_DOORBELL_PORT6_RANGE_OFFSET, + doorbell_index); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, + S2A_DOORBELL_PORT6_RANGE_SIZE, + doorbell_size); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, + S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE, + 0xe); + doorbell_range1 = REG_SET_FIELD(doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1, + S2A_DOORBELL_PORT6_TARGET_PORT_TYPE, + 0x3); + doorbell_range1 = REG_SET_FIELD(doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1, + S2A_DOORBELL_PORT6_TARGET_DIEID, + 0x0); + doorbell_range1 = REG_SET_FIELD(doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1, + S2A_DOORBELL_PORT6_TARGET_PORT_ID, + 0x0); + } + + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, doorbell_range); + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1, doorbell_range1); + } +} + +static void nbio_v6_3_2_vcn_doorbell_range(struct amdgpu_device *adev, + bool use_doorbell, int doorbell_index, + int instance) +{ + u32 doorbell_range = 0; + u32 doorbell_range1 = 0; + + if (use_doorbell) { + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, + S2A_DOORBELL_PORT2_ENABLE, + 0x1); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, + S2A_DOORBELL_PORT2_AWID, + (instance % adev->vcn.num_inst_per_aid) ? 0x7 : 0x4); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, + S2A_DOORBELL_PORT2_RANGE_OFFSET, + doorbell_index); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, + S2A_DOORBELL_PORT2_RANGE_SIZE, + 8); + doorbell_range = REG_SET_FIELD(doorbell_range, + GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, + S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE, + (instance % adev->vcn.num_inst_per_aid) ? 0x7 : 0x4); + doorbell_range1 = REG_SET_FIELD(doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1, + S2A_DOORBELL_PORT2_TARGET_PORT_TYPE, + 0x3); + doorbell_range1 = REG_SET_FIELD(doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1, + S2A_DOORBELL_PORT2_TARGET_DIEID, + (instance / adev->vcn.num_inst_per_aid) ? 0x3 : 0x0); + doorbell_range1 = REG_SET_FIELD(doorbell_range1, + GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1, + S2A_DOORBELL_PORT2_TARGET_PORT_ID, + 0x0); + } + + switch (instance) { + case 0: + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, doorbell_range); + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1, doorbell_range1); + break; + case 1: + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL, doorbell_range); + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1, doorbell_range1); + break; + case 2: + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL, doorbell_range); + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1, doorbell_range1); + break; + case 3: + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range); + WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1, doorbell_range1); + break; + default: + dev_err(adev->dev, + "amdgpu: invalid vcn instance set when program doorbell range\n"); + break; + } +} + +static int nbio_v6_3_2_get_compute_partition_mode(struct amdgpu_device *adev) +{ + u32 tmp, px; + + tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_COMPUTE_STATUS); + px = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_COMPUTE_STATUS, + PARTITION_MODE); + + return px; +} + +static bool nbio_v6_3_2_is_nps_switch_requested(struct amdgpu_device *adev) +{ + u32 tmp; + + tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS); + tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS, + CHANGE_STATUS); + + /* 0x8 - NPS switch requested */ + return (tmp == 0x8); +} +static u32 nbio_v6_3_2_get_memory_partition_mode(struct amdgpu_device *adev, + u32 *supp_modes) +{ + u32 tmp; + + tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS); + tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS, NPS_MODE); + + if (supp_modes) { + *supp_modes = + RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_CAP); + } + + return ffs(tmp); +} + +const struct amdgpu_nbio_funcs nbio_v6_3_2_funcs = { + .get_pcie_index_offset = nbio_v6_3_2_get_pcie_index_offset, + .get_pcie_data_offset = nbio_v6_3_2_get_pcie_data_offset, + .get_pcie_index_hi_offset = nbio_v6_3_2_get_pcie_index_hi_offset, + .get_rev_id = nbio_v6_3_2_get_rev_id, + .mc_access_enable = nbio_v6_3_2_mc_access_enable, + .get_memsize = nbio_v6_3_2_get_memsize, + .enable_doorbell_aperture = nbio_v6_3_2_enable_doorbell_aperture, + .enable_doorbell_selfring_aperture = nbio_v6_3_2_enable_doorbell_selfring_aperture, + .enable_doorbell_interrupt = nbio_v6_3_2_enable_doorbell_interrupt, + .get_compute_partition_mode = nbio_v6_3_2_get_compute_partition_mode, + .get_memory_partition_mode = nbio_v6_3_2_get_memory_partition_mode, + .is_nps_switch_requested = nbio_v6_3_2_is_nps_switch_requested, + .ih_control = nbio_v6_3_2_ih_control, + .ih_doorbell_range = nbio_v6_3_2_ih_doorbell_range, + .gc_doorbell_init = nbio_v6_3_2_gc_doorbell_init, + .sdma_doorbell_range = nbio_v6_3_2_sdma_doorbell_range, + .vcn_doorbell_range = nbio_v6_3_2_vcn_doorbell_range, + .init_registers = nbio_v6_3_2_init_registers, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h new file mode 100644 index 000000000000..bc7f747c88f4 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h @@ -0,0 +1,31 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __NBIO_V6_3_2_H__ +#define __NBIO_V6_3_2_H__ + +#include "soc15_common.h" + +extern const struct amdgpu_nbio_funcs nbio_v6_3_2_funcs; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c index 7ce1a1b95606..72edf5326b05 100644 --- a/drivers/gpu/drm/amd/amdgpu/nv.c +++ b/drivers/gpu/drm/amd/amdgpu/nv.c @@ -354,29 +354,12 @@ static struct soc15_allowed_register_entry nv_allowed_read_registers[] = { { SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)}, }; -static uint32_t nv_read_indexed_register(struct amdgpu_device *adev, u32 se_num, - u32 sh_num, u32 reg_offset) -{ - uint32_t val; - - mutex_lock(&adev->grbm_idx_mutex); - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0); - - val = RREG32(reg_offset); - - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); - mutex_unlock(&adev->grbm_idx_mutex); - return val; -} - static uint32_t nv_get_register_value(struct amdgpu_device *adev, bool indexed, u32 se_num, u32 sh_num, u32 reg_offset) { if (indexed) { - return nv_read_indexed_register(adev, se_num, sh_num, reg_offset); + return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset); } else { if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG)) return adev->gfx.config.gb_addr_config; @@ -511,16 +494,6 @@ static int nv_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk) return 0; } -static void nv_program_aspm(struct amdgpu_device *adev) -{ - if (!amdgpu_device_should_use_aspm(adev)) - return; - - if (adev->nbio.funcs->program_aspm) - adev->nbio.funcs->program_aspm(adev); - -} - const struct amdgpu_ip_block_version nv_common_ip_block = { .type = AMD_IP_BLOCK_TYPE_COMMON, .major = 1, @@ -984,7 +957,7 @@ static int nv_common_hw_init(struct amdgpu_ip_block *ip_block) adev->nbio.funcs->apply_l1_link_width_reconfig_wa(adev); /* enable aspm */ - nv_program_aspm(adev); + amdgpu_nbio_program_aspm(adev); /* setup nbio registers */ adev->nbio.funcs->init_registers(adev); /* remap HDP registers to a hole in mmio space, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h index e8f768638fd5..ac34bac3c839 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h +++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h @@ -107,6 +107,7 @@ enum psp_gfx_cmd_id GFX_CMD_ID_CONFIG_SQ_PERFMON = 0x00000046, /* Config CGTT_SQ_CLK_CTRL */ /* Dynamic memory partitioninig (NPS mode change)*/ GFX_CMD_ID_FB_NPS_MODE = 0x00000048, /* Configure memory partitioning mode */ + GFX_CMD_ID_PERF_HW = 0x0000004C, /* performance monitor */ GFX_CMD_ID_FB_FW_RESERV_ADDR = 0x00000050, /* Query FW reservation addr */ GFX_CMD_ID_FB_FW_RESERV_EXT_ADDR = 0x00000051, /* Query FW reservation extended addr */ }; @@ -373,6 +374,13 @@ struct psp_gfx_cmd_fb_memory_part { uint32_t resvd; }; +struct psp_gfx_cmd_req_perf_hw { + uint32_t req; + uint32_t ptl_state; + uint32_t pref_format1; + uint32_t pref_format2; +}; + /* All GFX ring buffer commands. */ union psp_gfx_commands { @@ -389,6 +397,7 @@ union psp_gfx_commands struct psp_gfx_cmd_sriov_spatial_part cmd_spatial_part; struct psp_gfx_cmd_config_sq_perfmon config_sq_perfmon; struct psp_gfx_cmd_fb_memory_part cmd_memory_part; + struct psp_gfx_cmd_req_perf_hw cmd_req_perf_hw; }; struct psp_gfx_uresp_reserved @@ -415,12 +424,20 @@ struct psp_gfx_uresp_fw_reserve_info { uint32_t reserve_size; }; +struct psp_gfx_uresp_perf_hw { + uint32_t resp; + uint32_t ptl_state; + uint32_t pref_format1; + uint32_t pref_format2; +}; + /* Union of command-specific responses for GPCOM ring. */ union psp_gfx_uresp { struct psp_gfx_uresp_reserved reserved; struct psp_gfx_uresp_bootcfg boot_cfg; struct psp_gfx_uresp_fwar_db_info fwar_db_info; struct psp_gfx_uresp_fw_reserve_info fw_reserve_info; + struct psp_gfx_uresp_perf_hw perf_hw_info; }; /* Structure of GFX Response buffer. diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index fb7aaf5ae05c..479690c44f0d 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -217,7 +217,9 @@ static int psp_v11_0_bootloader_load_component(struct psp_context *psp, return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -263,7 +265,9 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c index c3cae29eeca1..f823f042788d 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c @@ -87,7 +87,9 @@ static int psp_v12_0_bootloader_load_sysdrv(struct psp_context *psp) return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + ret = psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -123,7 +125,9 @@ static int psp_v12_0_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c index a0c84f81c0c9..00b4a34e6601 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c @@ -271,10 +271,9 @@ static int psp_v13_0_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, @@ -353,10 +352,9 @@ static int psp_v13_0_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c index 5f39a2edcc95..3d5e26b3fa00 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c @@ -105,10 +105,9 @@ static int psp_v13_0_4_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, @@ -168,10 +167,9 @@ static int psp_v13_0_4_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c index 38dfc5c19f2a..040a61aefa86 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c @@ -140,10 +140,9 @@ static int psp_v14_0_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_36, @@ -214,10 +213,9 @@ static int psp_v14_0_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c index 2a8582e87f2b..2a4d91368ac6 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c @@ -33,6 +33,8 @@ MODULE_FIRMWARE("amdgpu/psp_15_0_0_toc.bin"); MODULE_FIRMWARE("amdgpu/psp_15_0_0_ta.bin"); +MODULE_FIRMWARE("amdgpu/psp_15_0_9_toc.bin"); +MODULE_FIRMWARE("amdgpu/psp_15_0_9_ta.bin"); static int psp_v15_0_0_init_microcode(struct psp_context *psp) { diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c index b2d7cbd894c0..ec20cd5eb755 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c @@ -32,6 +32,7 @@ #include "mp/mp_15_0_8_sh_mask.h" MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc.bin"); +MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc_1.bin"); static int psp_v15_0_8_init_microcode(struct psp_context *psp) { diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c index f5030efc6c80..24856c91c135 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c @@ -96,7 +96,9 @@ static int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp) return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + ret = psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -135,7 +137,9 @@ static int psp_v3_1_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 0090ace49024..93ec52c1f367 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -828,7 +828,6 @@ static int sdma_v2_4_early_init(struct amdgpu_ip_block *ip_block) return r; sdma_v2_4_set_ring_funcs(adev); - sdma_v2_4_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v2_4_vm_pte_funcs); sdma_v2_4_set_irq_funcs(adev); @@ -898,7 +897,9 @@ static int sdma_v2_4_hw_init(struct amdgpu_ip_block *ip_block) if (r) return r; - return r; + sdma_v2_4_set_buffer_funcs(adev); + + return 0; } static int sdma_v2_4_hw_fini(struct amdgpu_ip_block *ip_block) @@ -1235,8 +1236,7 @@ static const struct amdgpu_buffer_funcs sdma_v2_4_buffer_funcs = { static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v2_4_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v2_4_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v2_4_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 2526d393162a..3fde9be74690 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -1108,7 +1108,6 @@ static int sdma_v3_0_early_init(struct amdgpu_ip_block *ip_block) return r; sdma_v3_0_set_ring_funcs(adev); - sdma_v3_0_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v3_0_vm_pte_funcs); sdma_v3_0_set_irq_funcs(adev); @@ -1184,7 +1183,9 @@ static int sdma_v3_0_hw_init(struct amdgpu_ip_block *ip_block) if (r) return r; - return r; + sdma_v3_0_set_buffer_funcs(adev); + + return 0; } static int sdma_v3_0_hw_fini(struct amdgpu_ip_block *ip_block) @@ -1677,8 +1678,7 @@ static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = { static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v3_0_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v3_0_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v3_0_ip_block = diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index e64f2f6df9a9..cb64d17000df 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -1775,7 +1775,6 @@ static int sdma_v4_0_early_init(struct amdgpu_ip_block *ip_block) adev->sdma.has_page_queue = true; sdma_v4_0_set_ring_funcs(adev); - sdma_v4_0_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v4_0_vm_pte_funcs); sdma_v4_0_set_irq_funcs(adev); sdma_v4_0_set_ras_funcs(adev); @@ -1961,6 +1960,7 @@ static int sdma_v4_0_sw_fini(struct amdgpu_ip_block *ip_block) static int sdma_v4_0_hw_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; + int r; if (adev->flags & AMD_IS_APU) amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_SDMA, false, 0); @@ -1968,7 +1968,12 @@ static int sdma_v4_0_hw_init(struct amdgpu_ip_block *ip_block) if (!amdgpu_sriov_vf(adev)) sdma_v4_0_init_golden_registers(adev); - return sdma_v4_0_start(adev); + r = sdma_v4_0_start(adev); + if (r) + return r; + sdma_v4_0_set_buffer_funcs(adev); + + return 0; } static int sdma_v4_0_hw_fini(struct amdgpu_ip_block *ip_block) @@ -2626,13 +2631,9 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_buffer_funcs = { static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev) { if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= IP_VERSION(4, 4, 0)) - adev->mman.buffer_funcs = &sdma_v4_4_buffer_funcs; - else - adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs; - if (adev->sdma.has_page_queue) - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_buffer_funcs); else - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_0_buffer_funcs); } static void sdma_v4_0_get_ras_error_count(uint32_t value, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c index 78bdfed0a7fd..8652928861ad 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c @@ -457,7 +457,7 @@ static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 /* write the fence */ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -467,7 +467,7 @@ static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 addr += 4; amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); @@ -1368,7 +1368,6 @@ static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block) adev->sdma.has_page_queue = true; sdma_v4_4_2_set_ring_funcs(adev); - sdma_v4_4_2_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v4_4_2_vm_pte_funcs); sdma_v4_4_2_set_irq_funcs(adev); sdma_v4_4_2_set_ras_funcs(adev); @@ -1568,8 +1567,11 @@ static int sdma_v4_4_2_hw_init(struct amdgpu_ip_block *ip_block) sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask); r = sdma_v4_4_2_inst_start(adev, inst_mask, false); + if (r) + return r; + sdma_v4_4_2_set_buffer_funcs(adev); - return r; + return 0; } static int sdma_v4_4_2_hw_fini(struct amdgpu_ip_block *ip_block) @@ -2316,11 +2318,7 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = { static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs; - if (adev->sdma.has_page_queue) - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page; - else - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_2_buffer_funcs); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index 52f4e9e099cb..b809942b1eb7 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -527,7 +527,7 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -538,7 +538,7 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); @@ -1373,7 +1373,6 @@ static int sdma_v5_0_early_init(struct amdgpu_ip_block *ip_block) return r; sdma_v5_0_set_ring_funcs(adev); - sdma_v5_0_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v5_0_vm_pte_funcs); sdma_v5_0_set_irq_funcs(adev); sdma_v5_0_set_mqd_funcs(adev); @@ -1472,8 +1471,11 @@ static int sdma_v5_0_hw_init(struct amdgpu_ip_block *ip_block) sdma_v5_0_init_golden_registers(adev); r = sdma_v5_0_start(adev); + if (r) + return r; + sdma_v5_0_set_buffer_funcs(adev); - return r; + return 0; } static int sdma_v5_0_hw_fini(struct amdgpu_ip_block *ip_block) @@ -2052,10 +2054,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_0_buffer_funcs = { static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev) { - if (adev->mman.buffer_funcs == NULL) { - adev->mman.buffer_funcs = &sdma_v5_0_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; - } + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_0_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v5_0_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index b4fb90cc8f7d..87c1e29fd298 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -377,7 +377,7 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -388,7 +388,7 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); @@ -1264,7 +1264,6 @@ static int sdma_v5_2_early_init(struct amdgpu_ip_block *ip_block) return r; sdma_v5_2_set_ring_funcs(adev); - sdma_v5_2_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v5_2_vm_pte_funcs); sdma_v5_2_set_irq_funcs(adev); sdma_v5_2_set_mqd_funcs(adev); @@ -1385,8 +1384,14 @@ static int sdma_v5_2_sw_fini(struct amdgpu_ip_block *ip_block) static int sdma_v5_2_hw_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; + int r; + + r = sdma_v5_2_start(adev); + if (r) + return r; + sdma_v5_2_set_buffer_funcs(adev); - return sdma_v5_2_start(adev); + return 0; } static int sdma_v5_2_hw_fini(struct amdgpu_ip_block *ip_block) @@ -2056,10 +2061,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = { static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev) { - if (adev->mman.buffer_funcs == NULL) { - adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; - } + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_2_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v5_2_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index 8ca46e1e474e..d7537888e60c 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -55,6 +55,7 @@ MODULE_FIRMWARE("amdgpu/sdma_6_1_1.bin"); MODULE_FIRMWARE("amdgpu/sdma_6_1_2.bin"); MODULE_FIRMWARE("amdgpu/sdma_6_1_3.bin"); MODULE_FIRMWARE("amdgpu/sdma_6_1_4.bin"); +MODULE_FIRMWARE("amdgpu/sdma_6_4_0.bin"); #define SDMA1_REG_OFFSET 0x600 #define SDMA0_HYP_DEC_REG_START 0x5880 @@ -360,7 +361,7 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -371,7 +372,7 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); @@ -1313,7 +1314,6 @@ static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block) return r; sdma_v6_0_set_ring_funcs(adev); - sdma_v6_0_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v6_0_vm_pte_funcs); sdma_v6_0_set_irq_funcs(adev); sdma_v6_0_set_mqd_funcs(adev); @@ -1477,6 +1477,7 @@ static int sdma_v6_0_hw_init(struct amdgpu_ip_block *ip_block) r = sdma_v6_0_start(adev); if (r) return r; + sdma_v6_0_set_buffer_funcs(adev); return sdma_v6_0_set_userq_trap_interrupts(adev, true); } @@ -1886,8 +1887,7 @@ static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = { static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v6_0_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v6_0_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v6_0_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index 37191e2918d4..49c57a38151b 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -363,7 +363,7 @@ static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -374,7 +374,7 @@ static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); @@ -1299,7 +1299,6 @@ static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block) } sdma_v7_0_set_ring_funcs(adev); - sdma_v7_0_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_0_vm_pte_funcs); sdma_v7_0_set_irq_funcs(adev); sdma_v7_0_set_mqd_funcs(adev); @@ -1432,6 +1431,7 @@ static int sdma_v7_0_hw_init(struct amdgpu_ip_block *ip_block) r = sdma_v7_0_start(adev); if (r) return r; + sdma_v7_0_set_buffer_funcs(adev); return sdma_v7_0_set_userq_trap_interrupts(adev, true); } @@ -1836,8 +1836,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = { static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v7_0_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_0_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v7_0_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c index 9c9bbe043a47..b06001f6b536 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c @@ -331,7 +331,7 @@ static void sdma_v7_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -342,7 +342,7 @@ static void sdma_v7_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); @@ -1287,7 +1287,6 @@ static int sdma_v7_1_early_init(struct amdgpu_ip_block *ip_block) } sdma_v7_1_set_ring_funcs(adev); - sdma_v7_1_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_1_vm_pte_funcs); sdma_v7_1_set_irq_funcs(adev); sdma_v7_1_set_mqd_funcs(adev); @@ -1387,10 +1386,16 @@ static int sdma_v7_1_hw_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; uint32_t inst_mask; + int r; inst_mask = GENMASK(adev->sdma.num_instances - 1, 0); - return sdma_v7_1_inst_start(adev, inst_mask); + r = sdma_v7_1_inst_start(adev, inst_mask); + if (r) + return r; + sdma_v7_1_set_buffer_funcs(adev); + + return 0; } static int sdma_v7_1_hw_fini(struct amdgpu_ip_block *ip_block) @@ -1777,8 +1782,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_1_buffer_funcs = { static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &sdma_v7_1_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_1_buffer_funcs); } const struct amdgpu_ip_block_version sdma_v7_1_ip_block = { diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c index 3e58feb2d5e4..549708075eb4 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dma.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c @@ -487,7 +487,6 @@ static int si_dma_early_init(struct amdgpu_ip_block *ip_block) adev->sdma.num_instances = SDMA_MAX_INSTANCE; si_dma_set_ring_funcs(adev); - si_dma_set_buffer_funcs(adev); amdgpu_sdma_set_vm_pte_scheds(adev, &si_dma_vm_pte_funcs); si_dma_set_irq_funcs(adev); @@ -543,8 +542,14 @@ static int si_dma_sw_fini(struct amdgpu_ip_block *ip_block) static int si_dma_hw_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; + int r; - return si_dma_start(adev); + r = si_dma_start(adev); + if (r) + return r; + si_dma_set_buffer_funcs(adev); + + return 0; } static int si_dma_hw_fini(struct amdgpu_ip_block *ip_block) @@ -833,8 +838,7 @@ static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = { static void si_dma_set_buffer_funcs(struct amdgpu_device *adev) { - adev->mman.buffer_funcs = &si_dma_buffer_funcs; - adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; + amdgpu_sdma_set_buffer_funcs_scheds(adev, &si_dma_buffer_funcs); } const struct amdgpu_ip_block_version si_dma_ip_block = diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index b456e4541d9a..87b398dd0769 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -401,29 +401,12 @@ static struct soc15_allowed_register_entry soc15_allowed_read_registers[] = { { SOC15_REG_ENTRY(GC, 0, mmDB_DEBUG2)}, }; -static uint32_t soc15_read_indexed_register(struct amdgpu_device *adev, u32 se_num, - u32 sh_num, u32 reg_offset) -{ - uint32_t val; - - mutex_lock(&adev->grbm_idx_mutex); - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0); - - val = RREG32(reg_offset); - - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); - mutex_unlock(&adev->grbm_idx_mutex); - return val; -} - static uint32_t soc15_get_register_value(struct amdgpu_device *adev, bool indexed, u32 se_num, u32 sh_num, u32 reg_offset) { if (indexed) { - return soc15_read_indexed_register(adev, se_num, sh_num, reg_offset); + return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset); } else { if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG)) return adev->gfx.config.gb_addr_config; @@ -695,15 +678,6 @@ static int soc15_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk return 0; } -static void soc15_program_aspm(struct amdgpu_device *adev) -{ - if (!amdgpu_device_should_use_aspm(adev)) - return; - - if (adev->nbio.funcs->program_aspm) - adev->nbio.funcs->program_aspm(adev); -} - const struct amdgpu_ip_block_version vega10_common_ip_block = { .type = AMD_IP_BLOCK_TYPE_COMMON, @@ -1284,7 +1258,7 @@ static int soc15_common_hw_init(struct amdgpu_ip_block *ip_block) struct amdgpu_device *adev = ip_block->adev; /* enable aspm */ - soc15_program_aspm(adev); + amdgpu_nbio_program_aspm(adev); /* setup nbio registers */ adev->nbio.funcs->init_registers(adev); /* remap HDP registers to a hole in mmio space, diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index fbd1d97f33ad..e0b80abcd075 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -306,29 +306,12 @@ static struct soc15_allowed_register_entry soc21_allowed_read_registers[] = { { SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG)}, }; -static uint32_t soc21_read_indexed_register(struct amdgpu_device *adev, u32 se_num, - u32 sh_num, u32 reg_offset) -{ - uint32_t val; - - mutex_lock(&adev->grbm_idx_mutex); - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0); - - val = RREG32(reg_offset); - - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); - mutex_unlock(&adev->grbm_idx_mutex); - return val; -} - static uint32_t soc21_get_register_value(struct amdgpu_device *adev, bool indexed, u32 se_num, u32 sh_num, u32 reg_offset) { if (indexed) { - return soc21_read_indexed_register(adev, se_num, sh_num, reg_offset); + return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset); } else { if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG) && adev->gfx.config.gb_addr_config) return adev->gfx.config.gb_addr_config; @@ -423,6 +406,7 @@ soc21_asic_reset_method(struct amdgpu_device *adev) case IP_VERSION(14, 0, 4): case IP_VERSION(14, 0, 5): case IP_VERSION(15, 0, 0): + case IP_VERSION(15, 0, 9): return AMD_RESET_METHOD_MODE2; default: if (amdgpu_dpm_is_baco_supported(adev)) @@ -470,15 +454,6 @@ static int soc21_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk return 0; } -static void soc21_program_aspm(struct amdgpu_device *adev) -{ - if (!amdgpu_device_should_use_aspm(adev)) - return; - - if (adev->nbio.funcs->program_aspm) - adev->nbio.funcs->program_aspm(adev); -} - const struct amdgpu_ip_block_version soc21_common_ip_block = { .type = AMD_IP_BLOCK_TYPE_COMMON, .major = 1, @@ -859,6 +834,65 @@ static int soc21_common_early_init(struct amdgpu_ip_block *ip_block) AMD_PG_SUPPORT_GFX_PG; adev->external_rev_id = adev->rev_id + 0x1; break; + case IP_VERSION(11, 5, 6): + adev->cg_flags = 0; + adev->pg_flags = 0; + adev->external_rev_id = adev->rev_id + 0xd0; + break; + case IP_VERSION(11, 7, 0): + adev->cg_flags = AMD_CG_SUPPORT_VCN_MGCG | + AMD_CG_SUPPORT_JPEG_MGCG | + AMD_CG_SUPPORT_GFX_CGCG | + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_FGCG | + AMD_CG_SUPPORT_REPEATER_FGCG | + AMD_CG_SUPPORT_GFX_PERF_CLK | + AMD_CG_SUPPORT_GFX_3D_CGCG | + AMD_CG_SUPPORT_GFX_3D_CGLS | + AMD_CG_SUPPORT_MC_MGCG | + AMD_CG_SUPPORT_MC_LS | + AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_DS | + AMD_CG_SUPPORT_HDP_SD | + AMD_CG_SUPPORT_ATHUB_MGCG | + AMD_CG_SUPPORT_ATHUB_LS | + AMD_CG_SUPPORT_IH_CG | + AMD_CG_SUPPORT_BIF_MGCG | + AMD_CG_SUPPORT_BIF_LS; + adev->pg_flags = AMD_PG_SUPPORT_VCN_DPG | + AMD_PG_SUPPORT_VCN | + AMD_PG_SUPPORT_JPEG | + AMD_PG_SUPPORT_GFX_PG; + adev->external_rev_id = adev->rev_id + 0xF; + break; + case IP_VERSION(11, 7, 1): + adev->cg_flags = AMD_CG_SUPPORT_VCN_MGCG | + AMD_CG_SUPPORT_JPEG_MGCG | + AMD_CG_SUPPORT_GFX_CGCG | + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_GFX_MGCG | + AMD_CG_SUPPORT_GFX_FGCG | + AMD_CG_SUPPORT_REPEATER_FGCG | + AMD_CG_SUPPORT_GFX_PERF_CLK | + AMD_CG_SUPPORT_GFX_3D_CGCG | + AMD_CG_SUPPORT_GFX_3D_CGLS | + AMD_CG_SUPPORT_MC_MGCG | + AMD_CG_SUPPORT_MC_LS | + AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_DS | + AMD_CG_SUPPORT_HDP_SD | + AMD_CG_SUPPORT_ATHUB_MGCG | + AMD_CG_SUPPORT_ATHUB_LS | + AMD_CG_SUPPORT_IH_CG | + AMD_CG_SUPPORT_BIF_MGCG | + AMD_CG_SUPPORT_BIF_LS; + adev->pg_flags = AMD_PG_SUPPORT_VCN_DPG | + AMD_PG_SUPPORT_VCN | + AMD_PG_SUPPORT_JPEG | + AMD_PG_SUPPORT_GFX_PG; + adev->external_rev_id = adev->rev_id + 0x40; + break; default: /* FIXME: not supported yet */ return -EINVAL; @@ -925,7 +959,7 @@ static int soc21_common_hw_init(struct amdgpu_ip_block *ip_block) struct amdgpu_device *adev = ip_block->adev; /* enable aspm */ - soc21_program_aspm(adev); + amdgpu_nbio_program_aspm(adev); /* setup nbio registers */ adev->nbio.funcs->init_registers(adev); /* remap HDP registers to a hole in mmio space, diff --git a/drivers/gpu/drm/amd/amdgpu/soc24.c b/drivers/gpu/drm/amd/amdgpu/soc24.c index d1adf19a51c4..9dce30d2bb8d 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc24.c +++ b/drivers/gpu/drm/amd/amdgpu/soc24.c @@ -132,31 +132,12 @@ static struct soc15_allowed_register_entry soc24_allowed_read_registers[] = { { SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG)}, }; -static uint32_t soc24_read_indexed_register(struct amdgpu_device *adev, - u32 se_num, - u32 sh_num, - u32 reg_offset) -{ - uint32_t val; - - mutex_lock(&adev->grbm_idx_mutex); - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0); - - val = RREG32(reg_offset); - - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); - mutex_unlock(&adev->grbm_idx_mutex); - return val; -} - static uint32_t soc24_get_register_value(struct amdgpu_device *adev, bool indexed, u32 se_num, u32 sh_num, u32 reg_offset) { if (indexed) { - return soc24_read_indexed_register(adev, se_num, sh_num, reg_offset); + return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset); } else { if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG) && adev->gfx.config.gb_addr_config) @@ -515,8 +496,36 @@ static int soc24_common_suspend(struct amdgpu_ip_block *ip_block) return soc24_common_hw_fini(ip_block); } +static bool soc24_need_reset_on_resume(struct amdgpu_device *adev) +{ + u32 sol_reg1, sol_reg2; + + /* Will reset for the following suspend abort cases. + * 1) Only reset dGPU side. + * 2) S3 suspend got aborted and TOS is active. + * As for dGPU suspend abort cases the SOL value + * will be kept as zero at this resume point. + */ + if (!(adev->flags & AMD_IS_APU) && adev->in_s3) { + sol_reg1 = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_81); + msleep(100); + sol_reg2 = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_81); + + return (sol_reg1 != sol_reg2); + } + + return false; +} + static int soc24_common_resume(struct amdgpu_ip_block *ip_block) { + struct amdgpu_device *adev = ip_block->adev; + + if (soc24_need_reset_on_resume(adev)) { + dev_info(adev->dev, "S3 suspend aborted, resetting..."); + soc24_asic_reset(adev); + } + return soc24_common_hw_init(ip_block); } diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c index 709b1669b07b..5f05c8e68297 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c @@ -184,31 +184,13 @@ static struct soc15_allowed_register_entry soc_v1_0_allowed_read_registers[] = { { SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG_1) }, }; -static uint32_t soc_v1_0_read_indexed_register(struct amdgpu_device *adev, - u32 se_num, - u32 sh_num, - u32 reg_offset) -{ - uint32_t val; - - mutex_lock(&adev->grbm_idx_mutex); - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0); - - val = RREG32(reg_offset); - - if (se_num != 0xffffffff || sh_num != 0xffffffff) - amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); - mutex_unlock(&adev->grbm_idx_mutex); - return val; -} static uint32_t soc_v1_0_get_register_value(struct amdgpu_device *adev, bool indexed, u32 se_num, u32 sh_num, u32 reg_offset) { if (indexed) { - return soc_v1_0_read_indexed_register(adev, se_num, sh_num, reg_offset); + return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset); } else { if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG_1) && adev->gfx.config.gb_addr_config) @@ -330,6 +312,8 @@ static int soc_v1_0_common_early_init(struct amdgpu_ip_block *ip_block) return -EINVAL; } + adev->nbio.funcs->init_registers(adev); + return 0; } @@ -737,15 +721,8 @@ static int soc_v1_0_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, num_xcc_per_xcp = __soc_v1_0_get_xcc_per_xcp(xcp_mgr, mode); if (adev->gfx.imu.funcs && - adev->gfx.imu.funcs->switch_compute_partition) { - ret = adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode); - if (ret) - goto out; - } - if (adev->gfx.imu.funcs && - adev->gfx.imu.funcs->init_mcm_addr_lut && - amdgpu_emu_mode) - adev->gfx.imu.funcs->init_mcm_addr_lut(adev); + adev->gfx.imu.funcs->switch_compute_partition) + adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode); /* Init info about new xcps */ *num_xcps = num_xcc / num_xcc_per_xcp; diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c index db505ab32fa0..14092150336a 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c @@ -285,7 +285,8 @@ static int umc_v12_0_convert_error_address(struct amdgpu_device *adev, struct ta_ras_query_address_output *addr_out, bool dump_addr) { - uint32_t col, col_lower, row, row_lower, row_high, bank; + uint32_t row = 0, row_lower = 0, row_high = 0; + uint32_t col = 0, col_lower = 0, bank = 0; uint32_t channel_index = 0, umc_inst = 0; uint32_t i, bit_num, retire_unit, *flip_bits; uint64_t soc_pa, column, err_addr; diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c index ff7269bafae8..894780669f9c 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c @@ -1927,14 +1927,17 @@ out: #define RENCODE_IB_PARAM_SESSION_INIT 0x00000003 /* return the offset in ib if id is found, -1 otherwise */ -static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start) +static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start, uint32_t *length) { int i; uint32_t len; for (i = start; (len = amdgpu_ib_get_value(ib, i)) >= 8; i += len / 4) { - if (amdgpu_ib_get_value(ib, i + 1) == id) + if (amdgpu_ib_get_value(ib, i + 1) == id) { + if (length) + *length = len; return i; + } } return -1; } @@ -1944,14 +1947,14 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, struct amdgpu_ib *ib) { struct amdgpu_ring *ring = amdgpu_job_ring(job); - uint32_t val; + uint32_t val, len; int idx = 0, sidx; /* The first instance can decode anything */ if (!ring->me) return 0; - while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx)) >= 0) { + while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx, &len)) >= 0) { val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */ if (val == RADEON_VCN_ENGINE_TYPE_DECODE) { uint32_t valid_buf_flag = amdgpu_ib_get_value(ib, idx + 6); @@ -1964,12 +1967,12 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, amdgpu_ib_get_value(ib, idx + 8); return vcn_v4_0_dec_msg(p, job, msg_buffer_addr); } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) { - sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx); + sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx, NULL); if (sidx >= 0 && amdgpu_ib_get_value(ib, sidx + 2) == RENCODE_ENCODE_STANDARD_AV1) return vcn_v4_0_limit_sched(p, job); } - idx += amdgpu_ib_get_value(ib, idx) / 4; + idx += len / 4; } return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c index 10e8fc2821f3..7f001c32e911 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c @@ -1686,7 +1686,7 @@ static int vcn_v4_0_3_reset_jpeg_post_helper(struct amdgpu_device *adev, int ins for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) { ring = &adev->jpeg.inst[inst].ring_dec[i]; /* Force completion of any remaining jobs */ - amdgpu_fence_driver_force_completion(ring); + amdgpu_fence_driver_force_completion(ring, NULL); if (ring->use_doorbell) WREG32_SOC15_OFFSET( diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c index d5f49fa33bee..45580e9c4e0c 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c @@ -1234,6 +1234,38 @@ static const struct amdgpu_ring_funcs vcn_v5_0_0_unified_ring_vm_funcs = { .reset = vcn_v5_0_0_ring_reset, }; +static const struct amdgpu_ring_funcs vcn_v5_0_0_unified_ring_vm_funcs_secure = { + .type = AMDGPU_RING_TYPE_VCN_ENC, + .align_mask = 0x3f, + .nop = VCN_ENC_CMD_NO_OP, + .secure_submission_supported = true, + .no_user_fence = true, + .get_rptr = vcn_v5_0_0_unified_ring_get_rptr, + .get_wptr = vcn_v5_0_0_unified_ring_get_wptr, + .set_wptr = vcn_v5_0_0_unified_ring_set_wptr, + .emit_frame_size = + SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + + SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + + 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ + 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ + 1, /* vcn_v2_0_enc_ring_insert_end */ + .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ + .emit_ib = vcn_v2_0_enc_ring_emit_ib, + .emit_fence = vcn_v2_0_enc_ring_emit_fence, + .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush, + .test_ring = amdgpu_vcn_enc_ring_test_ring, + .test_ib = amdgpu_vcn_unified_ring_test_ib, + .insert_nop = amdgpu_ring_insert_nop, + .insert_end = vcn_v2_0_enc_ring_insert_end, + .pad_ib = amdgpu_ring_generic_pad_ib, + .begin_use = amdgpu_vcn_ring_begin_use, + .end_use = amdgpu_vcn_ring_end_use, + .emit_wreg = vcn_v2_0_enc_ring_emit_wreg, + .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait, + .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, + .reset = vcn_v5_0_0_ring_reset, +}; + /** * vcn_v5_0_0_set_unified_ring_funcs - set unified ring functions * @@ -1249,7 +1281,12 @@ static void vcn_v5_0_0_set_unified_ring_funcs(struct amdgpu_device *adev) if (adev->vcn.harvest_config & (1 << i)) continue; - adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v5_0_0_unified_ring_vm_funcs; + if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 3, 0)) + adev->vcn.inst[i].ring_enc[0].funcs = + &vcn_v5_0_0_unified_ring_vm_funcs_secure; + else + adev->vcn.inst[i].ring_enc[0].funcs = + &vcn_v5_0_0_unified_ring_vm_funcs; adev->vcn.inst[i].ring_enc[0].me = i; } } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c index 54fbf8d73ca6..d3db0494341e 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c @@ -1332,7 +1332,7 @@ static int vcn_v5_0_1_reset_jpeg_post_helper(struct amdgpu_device *adev, int ins for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) { ring = &adev->jpeg.inst[inst].ring_dec[i]; /* Force completion of any remaining jobs */ - amdgpu_fence_driver_force_completion(ring); + amdgpu_fence_driver_force_completion(ring, NULL); if (ring->use_doorbell) WREG32_SOC15_OFFSET( diff --git a/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.c new file mode 100644 index 000000000000..c92c5fc59aeb --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.c @@ -0,0 +1,351 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/firmware.h> +#include "amdgpu.h" +#include "amdgpu_ucode.h" +#include "amdgpu_vpe.h" +#include "vpe_v2_0.h" +#include "soc15_common.h" +#include "ivsrcid/vpe/irqsrcs_vpe_6_1.h" +#include "vpe/vpe_2_0_0_offset.h" +#include "vpe/vpe_2_0_0_sh_mask.h" + +MODULE_FIRMWARE("amdgpu/vpe_2_0_0.bin"); +MODULE_FIRMWARE("amdgpu/vpe_2_2_0.bin"); + +#define VPE_THREAD1_UCODE_OFFSET 0x8000 + +static uint32_t vpe_v2_0_get_reg_offset(struct amdgpu_vpe *vpe, uint32_t inst, uint32_t offset) +{ + uint32_t base; + + base = vpe->ring.adev->reg_offset[VPE_HWIP][inst][0]; + + return base + offset; +} + +static int vpe_v2_0_irq_init(struct amdgpu_vpe *vpe) +{ + struct amdgpu_device *adev = container_of(vpe, struct amdgpu_device, vpe); + int ret; + + ret = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_VPE, + VPE_6_1_SRCID__VPE_TRAP, + &adev->vpe.trap_irq); + if (ret) + return ret; + + return 0; +} + +static int vpe_v2_0_load_microcode(struct amdgpu_vpe *vpe) +{ + struct amdgpu_device *adev = vpe->ring.adev; + const struct vpe_firmware_header_v1_0 *vpe_hdr; + const __le32 *data; + uint32_t ucode_offset[2], ucode_size[2], size_dw, ret; + uint32_t f32_offset, f32_cntl, reg_data; + + ret = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL)); + ret = REG_SET_FIELD(ret, VPEC_CNTL, UMSCH_INT_ENABLE, 0); + WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL), ret); + + reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL2)); + reg_data = REG_SET_FIELD(reg_data, VPEC_CNTL2, IB_FIFO_WATERMARK, 1); + WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL2), reg_data); + + if (amdgpu_vpe_configure_dpm(vpe)) + dev_warn(adev->dev, "VPE DPM not enabled.\n"); + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + + f32_offset = vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL); + f32_cntl = RREG32(f32_offset); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, HALT, 0); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_RESET, 0); + + adev->vpe.cmdbuf_cpu_addr[0] = f32_offset; + adev->vpe.cmdbuf_cpu_addr[1] = f32_cntl; + + return amdgpu_vpe_psp_update_sram(adev); + } + + /* Halt and Check F32 cleaness */ + f32_offset = vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL); + f32_cntl = RREG32(f32_offset); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, HALT, 1); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_RESET, 1); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_CHECKSUM_CLR, 1); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH0_CHECKSUM_CLR, 1); + WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL), f32_cntl); + + f32_cntl = RREG32(f32_offset); + if (!REG_GET_FIELD(f32_cntl, VPEC_F32_CNTL, HALT)) { + dev_err(adev->dev, "VPEC is not halted"); + return -EBUSY; + } + + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_CHECKSUM_CLR, 0); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH0_CHECKSUM_CLR, 0); + WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL), f32_cntl); + + reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_UCODE_CHECKSUM)); + if (reg_data) { + dev_err(adev->dev, "VPE FW checksum 0 not clean"); + return -EBUSY; + } + reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_UCODE1_CHECKSUM)); + if (reg_data) { + dev_err(adev->dev, "VPE FW checksum 1 not clean"); + return -EBUSY; + } + + reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_STATUS2)); + if (REG_GET_FIELD(reg_data, VPEC_STATUS2, TH0F32_INSTR_PTR)) { + dev_err(adev->dev, "VPE FW initial status not clean"); + return -EBUSY; + } + + reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_STATUS6)); + if (REG_GET_FIELD(reg_data, VPEC_STATUS6, TH1F32_INSTR_PTR)) { + dev_err(adev->dev, "VPE FW initial status not clean"); + return -EBUSY; + } + /* end of F32 cleaness check */ + + vpe_hdr = (const struct vpe_firmware_header_v1_0 *)adev->vpe.fw->data; + + /* Thread 0(command thread) ucode offset/size */ + ucode_offset[0] = le32_to_cpu(vpe_hdr->header.ucode_array_offset_bytes); + ucode_size[0] = le32_to_cpu(vpe_hdr->ctx_ucode_size_bytes); + /* Thread 1(control thread) ucode offset/size */ + ucode_offset[1] = le32_to_cpu(vpe_hdr->ctl_ucode_offset); + ucode_size[1] = le32_to_cpu(vpe_hdr->ctl_ucode_size_bytes); + + reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_PG_CNTL)); + reg_data = REG_SET_FIELD(reg_data, VPEC_PG_CNTL, PG_EN, 0); + WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_PG_CNTL), reg_data); + + for (int j = 0; j < vpe->num_instances; j++) { + for (int i = 0; i < 2; i++) { + if (i > 0) + WREG32(vpe_get_reg_offset(vpe, j, regVPEC_UCODE_ADDR), VPE_THREAD1_UCODE_OFFSET); + else + WREG32(vpe_get_reg_offset(vpe, j, regVPEC_UCODE_ADDR), 0); + + data = (const __le32 *)(adev->vpe.fw->data + ucode_offset[i]); + size_dw = ucode_size[i] / sizeof(__le32); + + while (size_dw--) { + if (amdgpu_emu_mode && size_dw % 500 == 0) + msleep(1); + WREG32(vpe_get_reg_offset(vpe, j, regVPEC_UCODE_DATA), le32_to_cpup(data++)); + } + } + } + + reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_PG_CNTL)); + reg_data = REG_SET_FIELD(reg_data, VPEC_PG_CNTL, PG_EN, 1); + WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_PG_CNTL), reg_data); + + /* Unhalt F32 */ + f32_cntl = RREG32(f32_offset); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, HALT, 0); + f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_RESET, 0); + WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL), f32_cntl); + + return 0; +} + +static int vpe_v2_0_ring_start(struct amdgpu_vpe *vpe) +{ + struct amdgpu_ring *ring = &vpe->ring; + struct amdgpu_device *adev = ring->adev; + uint32_t doorbell, doorbell_offset; + uint32_t rb_bufsz, rb_cntl; + uint32_t ib_cntl, i; + int ret; + + for (i = 0; i < vpe->num_instances; i++) { + /* Set ring buffer size in dwords */ + rb_bufsz = order_base_2(ring->ring_size / 4); + rb_cntl = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_CNTL)); + rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RB_SIZE, rb_bufsz); + rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RB_PRIV, 1); + rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RB_VMID, 0); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_CNTL), rb_cntl); + + /* Initialize the ring buffer's read and write pointers */ + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_RPTR), 0); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_RPTR_HI), 0); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_WPTR), 0); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_WPTR_HI), 0); + + /* set the wb address whether it's enabled or not */ + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_RPTR_ADDR_LO), + lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_RPTR_ADDR_HI), + upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF); + + rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1); + + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_BASE), ring->gpu_addr >> 8); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_BASE_HI), ring->gpu_addr >> 40); + + ring->wptr = 0; + + /* before programing wptr to a less value, need set minor_ptr_update first */ + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_MINOR_PTR_UPDATE), 1); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr) << 2); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2); + /* set minor_ptr_update to 0 after wptr programed */ + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_MINOR_PTR_UPDATE), 0); + + doorbell_offset = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_DOORBELL_OFFSET)); + doorbell_offset = REG_SET_FIELD(doorbell_offset, VPEC_QUEUE0_DOORBELL_OFFSET, OFFSET, ring->doorbell_index + i*4); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_DOORBELL_OFFSET), doorbell_offset); + + doorbell = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_DOORBELL)); + doorbell = REG_SET_FIELD(doorbell, VPEC_QUEUE0_DOORBELL, ENABLE, ring->use_doorbell ? 1 : 0); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_DOORBELL), doorbell); + + adev->nbio.funcs->vpe_doorbell_range(adev, i, ring->use_doorbell, ring->doorbell_index + i*4, 4); + + rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1); + rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RB_ENABLE, 1); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_CNTL), rb_cntl); + + ib_cntl = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_IB_CNTL)); + ib_cntl = REG_SET_FIELD(ib_cntl, VPEC_QUEUE0_IB_CNTL, IB_ENABLE, 1); + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_IB_CNTL), ib_cntl); + } + + ret = amdgpu_ring_test_helper(ring); + if (ret) + return ret; + + return 0; +} + +static int vpe_v2_0_ring_stop(struct amdgpu_vpe *vpe) +{ + struct amdgpu_device *adev = vpe->ring.adev; + uint32_t queue_reset, i; + int ret; + + for (i = 0; i < vpe->num_instances; i++) { + queue_reset = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE_RESET_REQ)); + + queue_reset = REG_SET_FIELD(queue_reset, VPEC_QUEUE_RESET_REQ, QUEUE0_RESET, 1); + + WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE_RESET_REQ), queue_reset); + /* timeout length is adev->timeout_usec */ + ret = SOC15_WAIT_ON_RREG(VPE, i, regVPEC_QUEUE_RESET_REQ, 0, + VPEC_QUEUE_RESET_REQ__QUEUE0_RESET_MASK); + + if (ret) + dev_err(adev->dev, "VPE queue reset failed\n"); + } + + vpe->ring.sched.ready = false; + + return ret; +} + +static int vpe_v2_0_set_trap_irq_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + unsigned int type, + enum amdgpu_interrupt_state state) +{ + struct amdgpu_vpe *vpe = &adev->vpe; + uint32_t vpe_cntl; + + vpe_cntl = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL)); + vpe_cntl = REG_SET_FIELD(vpe_cntl, VPEC_CNTL, TRAP_ENABLE, + state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); + + WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL), vpe_cntl); + + return 0; +} + +static int vpe_v2_0_process_trap_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + + DRM_DEBUG("IH: VPE trap\n"); + + switch (entry->client_id) { + case SOC21_IH_CLIENTID_VPE: + amdgpu_fence_process(&adev->vpe.ring); + break; + default: + break; + } + + return 0; +} + +static int vpe_v2_0_set_regs(struct amdgpu_vpe *vpe) +{ + vpe->regs.queue0_rb_rptr_lo = regVPEC_QUEUE0_RB_RPTR; + vpe->regs.queue0_rb_rptr_hi = regVPEC_QUEUE0_RB_RPTR_HI; + vpe->regs.queue0_rb_wptr_lo = regVPEC_QUEUE0_RB_WPTR; + vpe->regs.queue0_rb_wptr_hi = regVPEC_QUEUE0_RB_WPTR_HI; + vpe->regs.queue0_preempt = regVPEC_QUEUE0_PREEMPT; + vpe->regs.dpm_enable = regVPEC_PUB_DUMMY2; + + vpe->regs.dpm_pratio = regVPEC_QUEUE6_DUMMY4; + vpe->regs.dpm_request_interval = regVPEC_QUEUE5_DUMMY3; + vpe->regs.dpm_decision_threshold = regVPEC_QUEUE5_DUMMY4; + vpe->regs.dpm_busy_clamp_threshold = regVPEC_QUEUE7_DUMMY2; + vpe->regs.dpm_idle_clamp_threshold = regVPEC_QUEUE7_DUMMY3; + vpe->regs.dpm_request_lv = regVPEC_QUEUE7_DUMMY1; + vpe->regs.context_indicator = regVPEC_QUEUE6_DUMMY3; + + return 0; +} + +static struct vpe_funcs vpe_v2_0_funcs = { + .get_reg_offset = vpe_v2_0_get_reg_offset, + .set_regs = vpe_v2_0_set_regs, + .irq_init = vpe_v2_0_irq_init, + .init_microcode = amdgpu_vpe_init_microcode, + .load_microcode = vpe_v2_0_load_microcode, + .ring_init = amdgpu_vpe_ring_init, + .ring_start = vpe_v2_0_ring_start, + .ring_stop = vpe_v2_0_ring_stop, + .ring_fini = amdgpu_vpe_ring_fini, +}; + +static const struct amdgpu_irq_src_funcs vpe_v2_0_trap_irq_funcs = { + .set = vpe_v2_0_set_trap_irq_state, + .process = vpe_v2_0_process_trap_irq, +}; + +void vpe_v2_0_set_funcs(struct amdgpu_vpe *vpe) +{ + vpe->funcs = &vpe_v2_0_funcs; + vpe->trap_irq.funcs = &vpe_v2_0_trap_irq_funcs; +} diff --git a/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.h b/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.h new file mode 100644 index 000000000000..e9f2077bfdc2 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.h @@ -0,0 +1,29 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef __VPE_V2_0_H__ +#define __VPE_V2_0_H__ + +#include "amdgpu_vpe.h" + +void vpe_v2_0_set_funcs(struct amdgpu_vpe *vpe); + +#endif |