Merge branch 'drm-next' of https://gitlab.freedesktop.org/agd5f/linux.git

216 files changed, 5903 insertions, 2223 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 55b5d11f1938..8687afc00c24 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -92,7 +92,7 @@ amdgpu-y += \
 	sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o \
 	nbio_v7_9.o aqua_vanjaram.o nbio_v7_11.o lsdma_v7_0.o hdp_v7_0.o nbif_v6_3_1.o \
 	cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o nbio_v6_3_2.o nbio_v7_11_5.o \
-	nbif_v7_10.o hdp_v8_0.o lsdma_v8_0.o
+	nbif_v7_10.o hdp_v8_0.o lsdma_v8_0.o nbio_v7_11_4.o
 
 # add DF block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index ce671731b700..5a32b54b1e9b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -426,6 +426,9 @@ struct amdgpu_fpriv {
 
 	/** GPU partition selection */
 	uint32_t		xcp_id;
+
+	/** Signaled when a KFD interrupt drain fence for this PASID is seen */
+	struct completion	irq_drain_done;
 };
 
 int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
@@ -844,8 +847,17 @@ struct amdgpu_device {
 	struct mutex	mn_lock;
 	DECLARE_HASHTABLE(mn_hash, 7);
 
-	/* tracking pinned memory */
+	/* tracking pinned memory
+	 *
+	 *	vram_pin_size = sum(vram_pin_size_xcp[]) + vram_pin_size_common
+	 *
+	 * vram_pin_size         total pin size in device
+	 * vram_pin_size_xcp[i]  pins owned by partition i (bo->xcp_id = i, i >= 0)
+	 * vram_pin_size_common  pins not owned by any partition, bo->xcp_id = -1
+	 */
 	atomic64_t vram_pin_size;
+	atomic64_t vram_pin_size_xcp[MAX_XCP];
+	atomic64_t vram_pin_size_common;
 	atomic64_t visible_pin_size;
 	atomic64_t gart_pin_size;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 3391e8c77f13..e023cc734ebc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -789,7 +789,7 @@ void amdgpu_amdkfd_set_compute_idle(struct amdgpu_device *adev, bool idle)
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
 {
 	if (adev->kfd.dev)
-		return vmid >= adev->vm_manager.first_kfd_vmid;
+		return !!(adev->vm_manager.vmid_uq_mask_gfxhub & BIT(vmid));
 
 	return false;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index e3da382ceb3c..77c0504cc158 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -42,6 +42,12 @@
 
 extern uint64_t amdgpu_amdkfd_total_mem_size;
 
+/*
+ * Align VRAM availability to 2MB to avoid fragmentation caused by 4K
+ * allocations in the tail 2MB BO chunk.
+ */
+#define VRAM_AVAILABLITY_ALIGN (1 << 21)
+
 enum TLB_FLUSH_TYPE {
 	TLB_FLUSH_LEGACY = 0,
 	TLB_FLUSH_LIGHTWEIGHT,
@@ -106,6 +112,7 @@ struct amdgpu_kfd_dev {
 	struct kfd_dev *dev;
 	int64_t vram_used[MAX_XCP];
 	uint64_t vram_used_aligned[MAX_XCP];
+	atomic64_t vram_pinned[MAX_XCP];
 	bool init_complete;
 	struct work_struct reset_work;
 
@@ -389,6 +396,8 @@ void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
  * Allows KFD to release its resources associated with the GEM object.
  */
 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo);
+void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo, int xcp_id);
+void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo, int xcp_id);
 void amdgpu_amdkfd_reserve_system_mem(uint64_t size);
 #else
 static inline
@@ -406,6 +415,16 @@ static inline
 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
 {
 }
+
+static inline
+void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo, int xcp_id)
+{
+}
+
+static inline
+void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo, int xcp_id)
+{
+}
 #endif
 
 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
index 0694bff871d2..ec095003667f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
@@ -77,25 +77,21 @@ static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev,
 				unsigned int engine_id,
 				unsigned int queue_id)
 {
-	uint32_t sdma_engine_reg_base = 0;
-	uint32_t sdma_rlc_reg_offset;
 	uint32_t dev_inst = GET_INST(SDMA0, engine_id);
+	int xcc_id = adev->sdma.instance[engine_id].xcc_id;
+	int xcc_inst = dev_inst % adev->sdma.num_inst_per_xcc;
+	uint32_t sdma_engine_reg_base;
+	uint32_t sdma_rlc_reg_offset;
 
-	switch (dev_inst % adev->sdma.num_inst_per_xcc) {
-	case 0:
-		sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA0,
-				dev_inst / adev->sdma.num_inst_per_xcc,
-				regSDMA0_SDMA_QUEUE0_RB_CNTL) - regSDMA0_SDMA_QUEUE0_RB_CNTL;
-		break;
-	case 1:
-		sdma_engine_reg_base = SOC15_REG_OFFSET(SDMA1,
-				dev_inst / adev->sdma.num_inst_per_xcc,
-				regSDMA1_SDMA_QUEUE0_RB_CNTL) - regSDMA0_SDMA_QUEUE0_RB_CNTL;
-		break;
-	default:
-		WARN(1, "Invalid SDMA engine id %d\n", engine_id);
-		break;
-	}
+	/*
+	 * On SDMA v7 (gc_12_x) all SDMA queue registers live in the GC_HWIP
+	 * aperture. The second SDMA per XCC is reached by adding a fixed
+	 * offset (regSDMA1_* - regSDMA0_* == SDMA1_REG_OFFSET
+	 */
+	sdma_engine_reg_base = adev->reg_offset[GC_HWIP][xcc_id][0];
+	if (xcc_inst != 0)
+		sdma_engine_reg_base +=
+			regSDMA1_SDMA_QUEUE0_RB_CNTL - regSDMA0_SDMA_QUEUE0_RB_CNTL;
 
 	sdma_rlc_reg_offset = sdma_engine_reg_base
 		+ queue_id * (regSDMA0_SDMA_QUEUE1_RB_CNTL - regSDMA0_SDMA_QUEUE0_RB_CNTL);
@@ -111,7 +107,7 @@ static int hqd_dump_v12_1(struct amdgpu_device *adev,
 			uint32_t (**dump)[2], uint32_t *n_regs, uint32_t inst)
 {
 	uint32_t i = 0, reg;
-#define HQD_N_REGS 56
+#define HQD_N_REGS (regCP_HQD_PQ_WPTR_HI - regCP_MQD_BASE_ADDR + 1)
 #define DUMP_REG(addr) do {				\
 		if (WARN_ON_ONCE(i >= HQD_N_REGS))	\
 			break;				\
@@ -141,8 +137,7 @@ static int hqd_sdma_dump_v12_1(struct amdgpu_device *adev,
 			     uint32_t engine_id, uint32_t queue_id,
 			     uint32_t (**dump)[2], uint32_t *n_regs)
 {
-	uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,
-			engine_id, queue_id);
+	uint32_t sdma_rlc_reg_offset;
 	uint32_t i = 0, reg;
 
 	const uint32_t first_reg = regSDMA0_SDMA_QUEUE0_RB_CNTL;
@@ -150,6 +145,17 @@ static int hqd_sdma_dump_v12_1(struct amdgpu_device *adev,
 #undef HQD_N_REGS
 #define HQD_N_REGS (last_reg - first_reg + 1)
 
+	/*
+	 * dqm_debugfs_hqds() iterates engine_id over
+	 * node_id * get_num_all_sdma_engines()..+get_num_all_sdma_engines().
+	 * On partitioned devices that range can address a logical SDMA engine
+	 * beyond the instances that actually exist on this node; reject it
+	 */
+	if (engine_id >= adev->sdma.num_instances)
+		return -EINVAL;
+
+	sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, engine_id, queue_id);
+
 	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 9a7e14a3c719..ff77e8ae9fe6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -46,12 +46,6 @@
 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
 #define AMDGPU_RESERVE_MEM_LIMIT			(3UL << 29)
 
-/*
- * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
- * BO chunk
- */
-#define VRAM_AVAILABLITY_ALIGN (1 << 21)
-
 /* Impose limit on how much memory KFD can use */
 static struct {
 	uint64_t max_system_mem_limit;
@@ -73,6 +67,8 @@ static const char * const domain_bit_to_string[] = {
 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
 
 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
+static size_t __get_available_memory(struct amdgpu_device *adev,
+				     uint8_t xcp_id, int64_t vram_used);
 
 static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
 		struct kgd_mem *mem)
@@ -169,13 +165,8 @@ void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 		uint64_t size, u32 alloc_flag, int8_t xcp_id)
 {
-	uint64_t reserved_for_pt =
-		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0);
 	size_t system_mem_needed, ttm_mem_needed, vram_needed;
 	int ret = 0;
-	uint64_t vram_size = 0;
 
 	system_mem_needed = 0;
 	ttm_mem_needed = 0;
@@ -196,7 +187,6 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 		if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
 			return -EINVAL;
 
-		vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
 		if (adev->apu_prefer_gtt) {
 			system_mem_needed = size;
 			ttm_mem_needed = size;
@@ -232,14 +222,11 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 	 * VRAM check since ttm_mem_limit check already cover this allocation
 	 */
 
-	if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu)) {
-		uint64_t vram_available =
-			vram_size - reserved_for_pt - reserved_for_ras -
-			atomic64_read(&adev->vram_pin_size);
-		if (adev->kfd.vram_used[xcp_id] + vram_needed > vram_available) {
-			ret = -ENOMEM;
-			goto release;
-		}
+	if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu) &&
+	    vram_needed > __get_available_memory(adev, xcp_id,
+						 adev->kfd.vram_used[xcp_id])) {
+		ret = -ENOMEM;
+		goto release;
 	}
 
 	/* Update memory accounting by decreasing available system
@@ -318,6 +305,30 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
 	kfree(bo->kfd_bo);
 }
 
+void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo, int xcp_id)
+{
+	if (!bo->kfd_bo)
+		return;
+
+	if (WARN_ONCE(xcp_id < 0, "KFD BO pinned in VRAM with no partition"))
+		return;
+
+	atomic64_add(amdgpu_bo_size(bo),
+		     &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned[xcp_id]);
+}
+
+void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo, int xcp_id)
+{
+	if (!bo->kfd_bo)
+		return;
+
+	if (WARN_ONCE(xcp_id < 0, "KFD BO unpinned in VRAM with no partition"))
+		return;
+
+	atomic64_sub(amdgpu_bo_size(bo),
+		     &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned[xcp_id]);
+}
+
 /**
  * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information
  * about USERPTR or DOOREBELL or MMIO BO.
@@ -1536,6 +1547,7 @@ static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
 		pr_err("Error in Pinning BO to domain: %d\n", domain);
 
 	amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
+
 out:
 	amdgpu_bo_unreserve(bo);
 	return ret;
@@ -1558,6 +1570,7 @@ static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
 		return;
 
 	amdgpu_bo_unpin(bo);
+
 	amdgpu_bo_unreserve(bo);
 }
 
@@ -1661,8 +1674,9 @@ out_unlock:
 	return ret;
 }
 
-size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
-					  uint8_t xcp_id)
+/* Caller must hold kfd_mem_limit.mem_limit_lock. */
+static size_t __get_available_memory(struct amdgpu_device *adev,
+				     uint8_t xcp_id, int64_t vram_used)
 {
 	uint64_t reserved_for_pt =
 		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
@@ -1671,14 +1685,17 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
 	ssize_t available;
 	uint64_t vram_available, system_mem_available, ttm_mem_available;
 
-	spin_lock(&kfd_mem_limit.mem_limit_lock);
+	lockdep_assert_held(&kfd_mem_limit.mem_limit_lock);
+
 	if (adev->apu_prefer_gtt && !adev->gmc.is_app_apu)
 		vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
-			- adev->kfd.vram_used_aligned[xcp_id];
+			- vram_used;
 	else
 		vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
-			- adev->kfd.vram_used_aligned[xcp_id]
-			- atomic64_read(&adev->vram_pin_size)
+			- vram_used
+			- atomic64_read(&adev->vram_pin_size_xcp[xcp_id])
+			- atomic64_read(&adev->vram_pin_size_common)
+			+ atomic64_read(&adev->kfd.vram_pinned[xcp_id])
 			- reserved_for_pt
 			- reserved_for_ras;
 
@@ -1693,19 +1710,29 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
 
 		available = min3(system_mem_available, ttm_mem_available,
 				 vram_available);
-		available = ALIGN_DOWN(available, PAGE_SIZE);
 	} else {
-		available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);
+		available = vram_available;
 	}
 
-	spin_unlock(&kfd_mem_limit.mem_limit_lock);
-
 	if (available < 0)
 		available = 0;
 
 	return available;
 }
 
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
+					  uint8_t xcp_id)
+{
+	size_t available;
+
+	spin_lock(&kfd_mem_limit.mem_limit_lock);
+	available = __get_available_memory(adev, xcp_id,
+					   adev->kfd.vram_used_aligned[xcp_id]);
+	spin_unlock(&kfd_mem_limit.mem_limit_lock);
+
+	return available;
+}
+
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		struct amdgpu_device *adev, uint64_t va, uint64_t size,
 		void *drm_priv, struct kgd_mem **mem,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5558e7330cf4..1d3c3ad9813e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -1157,6 +1157,20 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 			return r;
 	}
 
+	if (fpriv->seq64_va) {
+		bo_va = fpriv->seq64_va;
+		r = amdgpu_vm_bo_update(adev, bo_va, false);
+		if (r) {
+			DRM_ERROR("failed to do vm_bo_update on userq sem\n");
+			return r;
+		}
+
+		r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,
+				      GFP_KERNEL);
+		if (r)
+			return r;
+	}
+
 	/* FIXME: In theory this loop shouldn't be needed any more when
 	 * amdgpu_vm_handle_moved handles all moved BOs that are reserved
 	 * with p->ticket. But removing it caused test regressions, so I'm
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 48279392d385..f147c0c459da 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -593,10 +593,8 @@ bool amdgpu_device_supports_boco(struct amdgpu_device *adev)
 	if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
 		return false;
 
-	if (adev->has_pr3 ||
-	    ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
-		return true;
-	return false;
+	/* BOCO needs a platform power resource (_PR3) to cut the power */
+	return adev->has_pr3;
 }
 
 /**
@@ -2783,6 +2781,12 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
 
+	r = amdgpu_dpm_reapply_umd_profile(adev);
+	if (r) {
+		dev_err(adev->dev, "Failed to restore UMD profile state: %d\n", r);
+		return r;
+	}
+
 	amdgpu_device_fill_reset_magic(adev);
 
 	r = amdgpu_device_enable_mgpu_fan_boost();
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index 890792a94e09..c5dec95d135e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -45,6 +45,7 @@
 #include "nbio_v7_4.h"
 #include "nbio_v7_9.h"
 #include "nbio_v7_11.h"
+#include "nbio_v7_11_4.h"
 #include "nbio_v7_11_5.h"
 #include "hdp_v4_0.h"
 #include "vega10_ih.h"
@@ -3830,10 +3831,13 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		adev->nbio.hdp_flush_reg = &nbio_v7_7_hdp_flush_reg;
 		break;
 	case IP_VERSION(6, 3, 1):
-	case IP_VERSION(7, 11, 4):
 		adev->nbio.funcs = &nbif_v6_3_1_funcs;
 		adev->nbio.hdp_flush_reg = &nbif_v6_3_1_hdp_flush_reg;
 		break;
+	case IP_VERSION(7, 11, 4):
+		adev->nbio.funcs = &nbio_v7_11_4_funcs;
+		adev->nbio.hdp_flush_reg = &nbio_v7_11_4_hdp_flush_reg;
+		break;
 	case IP_VERSION(7, 11, 5):
 		adev->nbio.funcs = &nbio_v7_11_5_funcs;
 		adev->nbio.hdp_flush_reg = &nbio_v7_11_5_hdp_flush_reg;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index ab6f9131ac11..3e5634de077f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -2974,19 +2974,6 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev)
 	return 0;
 }
 
-static void amdgpu_restore_umd_profile_pstate_after_runpm(struct amdgpu_device *adev)
-{
-	enum amd_dpm_forced_level level;
-	uint32_t profile_mode_mask = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD |
-		AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK |
-		AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK |
-		AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
-
-	level = amdgpu_dpm_get_performance_level(adev);
-	if (level & profile_mode_mask)
-		amdgpu_asic_update_umd_stable_pstate(adev, true);
-}
-
 static int amdgpu_pmops_runtime_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
@@ -3032,7 +3019,6 @@ static int amdgpu_pmops_runtime_resume(struct device *dev)
 	if (adev->pm.rpm_mode == AMDGPU_RUNPM_PX)
 		drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
 
-	amdgpu_restore_umd_profile_pstate_after_runpm(adev);
 	adev->in_runpm = false;
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 3962b9f08dd6..4ed1706d8d5f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -752,22 +752,28 @@ int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
 		goto unreserve;
 	}
 
-	if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
+	switch (args->op) {
+	case AMDGPU_GEM_METADATA_OP_GET_METADATA:
 		amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
 		r = amdgpu_bo_get_metadata(robj, args->data.data,
 					   sizeof(args->data.data),
 					   &args->data.data_size_bytes,
 					   &args->data.flags);
-	} else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
+		break;
+	case AMDGPU_GEM_METADATA_OP_SET_METADATA:
 		if (args->data.data_size_bytes > sizeof(args->data.data)) {
 			r = -EINVAL;
-			goto unreserve;
+			break;
 		}
 		r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
 		if (!r)
 			r = amdgpu_bo_set_metadata(robj, args->data.data,
 						   args->data.data_size_bytes,
 						   args->data.flags);
+		break;
+	default:
+		r = -EINVAL;
+		break;
 	}
 
 unreserve:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 5d4276cd0f7d..4204c3c750a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -787,6 +787,30 @@ error_alloc:
 	dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
 }
 
+/**
+ * amdgpu_gmc_bind_vmid - bind a VMID's page table base and flush its TLB
+ *
+ * @adev: amdgpu device pointer
+ * @vmhub: vmhub the VMID belongs to
+ * @vmid: VMID to bind
+ * @pd_addr: new page directory address, 0 to unbind to a faulting address
+ *
+ * For VMIDs that are bound directly to a VM's page table outside the
+ * normal per-job VM flush path, e.g. a dedicated out-of-pool VMID.
+ */
+void amdgpu_gmc_bind_vmid(struct amdgpu_device *adev, unsigned int vmhub,
+			  unsigned int vmid, uint64_t pd_addr)
+{
+	bool is_mmhub = AMDGPU_IS_MMHUB0(vmhub) || AMDGPU_IS_MMHUB1(vmhub);
+
+	if (is_mmhub)
+		adev->mmhub.funcs->setup_vm_pt_regs(adev, vmid, pd_addr);
+	else
+		adev->gfxhub.funcs->setup_vm_pt_regs(adev, vmid, pd_addr);
+
+	amdgpu_gmc_flush_gpu_tlb(adev, vmid, vmhub, TLB_FLUSH_LEGACY);
+}
+
 static void amdgpu_gmc_flush_pasid_regs(struct amdgpu_device *adev, u16 pasid,
 					u32 flush_type, bool all_hub,
 					u32 inst)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 76e6f1f96913..d17055d4c50a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -444,6 +444,8 @@ int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev);
 void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 			      uint32_t vmhub, uint32_t flush_type);
+void amdgpu_gmc_bind_vmid(struct amdgpu_device *adev, unsigned int vmhub,
+			  unsigned int vmid, uint64_t pd_addr);
 int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, u16 pasid,
 				   u32 flush_type, bool all_hub, u32 inst);
 int amdgpu_gmc_flush_gpu_tlb_pasid_xccs(struct amdgpu_device *adev, u16 pasid,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 0f8b097001e3..f98a7c6797b3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -126,7 +126,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	struct amdgpu_fence *af;
 	struct amdgpu_fence *vm_af;
 	bool need_ctx_switch;
-	bool emit_spm_needed = false;
 	bool emit_gds_needed = false;
 	uint64_t fence_ctx;
 	uint32_t status = 0, alloc_size;
@@ -195,8 +194,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, struct amdgpu_job *job,
 
 	/* VM sequence */
 	vm_af->ib_wptr = ring->wptr;
-	amdgpu_vm_flush(ring, job, &need_pipe_sync, &emit_spm_needed,
-			&emit_gds_needed);
+	amdgpu_vm_flush(ring, job, &need_pipe_sync, &emit_gds_needed);
 	vm_af->ib_dw_size =
 		amdgpu_ring_get_dw_distance(ring, vm_af->ib_wptr, ring->wptr);
 
@@ -211,9 +209,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	if (need_pipe_sync)
 		amdgpu_ring_emit_pipeline_sync(ring);
 
-	if (emit_spm_needed)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, ring->xcc_id, ring, job->vmid);
-
 	if (emit_gds_needed)
 		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
 					    job->gds_size, job->gws_base,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
index 9cdd8870960a..1482fc031d44 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
@@ -172,6 +172,68 @@ struct amdgpu_fpriv *amdgpu_pasid_get_fpriv_locked(u32 pasid)
 }
 
 /**
+ * amdgpu_pasid_drain_irq - wait for pending KFD interrupts of a PASID
+ * @adev: amdgpu device the drain fence is sent to
+ * @pasid: PASID whose interrupts should be drained
+ * @ih_fence: IH entry injected as the drain fence
+ * @timeout: maximum time to wait, in jiffies
+ *
+ * Injects @ih_fence behind the interrupts already queued for KFD and waits
+ * until amdgpu_pasid_drain_irq_done() is called for @pasid.
+ *
+ * The caller must hold a reference to the DRM file that owns @pasid, so
+ * that its amdgpu_fpriv stays valid after the PASID lock is dropped.
+ *
+ * Returns:
+ * 0 on success or when no fence could be sent, -ENOENT if @pasid has no
+ * owner, -ETIMEDOUT on timeout or -ERESTARTSYS if interrupted.
+ */
+int amdgpu_pasid_drain_irq(struct amdgpu_device *adev, u32 pasid,
+			   u32 *ih_fence, unsigned long timeout)
+{
+	struct amdgpu_fpriv *fpriv;
+	unsigned long flags;
+	long r;
+
+	amdgpu_pasid_lock(&flags);
+	fpriv = amdgpu_pasid_get_fpriv_locked(pasid);
+	if (fpriv)
+		reinit_completion(&fpriv->irq_drain_done);
+	amdgpu_pasid_unlock(flags);
+	if (!fpriv)
+		return -ENOENT;
+
+	if (amdgpu_amdkfd_send_close_event_drain_irq(adev, ih_fence))
+		return 0;
+
+	r = wait_for_completion_interruptible_timeout(&fpriv->irq_drain_done,
+						      timeout);
+	if (!r)
+		return -ETIMEDOUT;
+
+	return r < 0 ? r : 0;
+}
+
+/**
+ * amdgpu_pasid_drain_irq_done - signal that a KFD interrupt drain completed
+ * @pasid: PASID carried by the drain fence
+ *
+ * Called by KFD when it processes the drain fence injected by
+ * amdgpu_pasid_drain_irq().
+ */
+void amdgpu_pasid_drain_irq_done(u32 pasid)
+{
+	struct amdgpu_fpriv *fpriv;
+	unsigned long flags;
+
+	amdgpu_pasid_lock(&flags);
+	fpriv = amdgpu_pasid_get_fpriv_locked(pasid);
+	if (fpriv)
+		complete(&fpriv->irq_drain_done);
+	amdgpu_pasid_unlock(flags);
+}
+
+/**
  * amdgpu_pasid_free_delayed - free pasid when fences signal
  *
  * @resv: reservation object with the fences to wait for
@@ -372,7 +434,6 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm,
 		return r;
 
 	job->vm_needs_flush = needs_flush;
-	job->spm_update_needed = true;
 	return 0;
 }
 
@@ -528,7 +589,7 @@ bool amdgpu_vmid_uses_reserved(struct amdgpu_vm *vm, unsigned int vmhub)
 }
 
 /*
- * amdgpu_vmid_alloc_reserved - reserve a specific VMID for this vm
+ * amdgpu_vmid_alloc_spm - reserve a specific VMID for this vm
  * @adev: amdgpu device structure
  * @vm: the VM to reserve an ID for
  * @vmhub: the VMHUB which should be used
@@ -537,7 +598,7 @@ bool amdgpu_vmid_uses_reserved(struct amdgpu_vm *vm, unsigned int vmhub)
  *
  * Returns: 0 for success, -ENOENT if an ID is already reserved.
  */
-int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+int amdgpu_vmid_alloc_spm(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			       unsigned vmhub)
 {
 	struct amdgpu_kq_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
@@ -551,9 +612,12 @@ int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		r = -ENOENT;
 		goto unlock;
 	}
-	/* Remove from normal round robin handling */
-	id = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vmid, list);
-	list_del_init(&id->list);
+	if (!adev->vm_manager.spm_vmid) {
+		r = -ENOENT;
+		goto unlock;
+	}
+
+	id = &id_mgr->ids[adev->vm_manager.spm_vmid];
 	vm->reserved_vmid[vmhub] = id;
 	id_mgr->reserved_vmid = true;
 	mutex_unlock(&id_mgr->lock);
@@ -565,22 +629,32 @@ unlock:
 }
 
 /*
- * amdgpu_vmid_free_reserved - free up a reserved VMID again
+ * amdgpu_vmid_free_spm - free up a reserved VMID again
  * @adev: amdgpu device structure
  * @vm: the VM with the reserved ID
  * @vmhub: the VMHUB which should be used
+ *
+ * Waits for all work that used the reserved VMID to finish before unbinding
+ * its page table, so submissions still in flight don't suddenly fault.
  */
-void amdgpu_vmid_free_reserved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+void amdgpu_vmid_free_spm(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			       unsigned vmhub)
 {
 	struct amdgpu_kq_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
+	struct amdgpu_vmid *id;
 
 	mutex_lock(&id_mgr->lock);
-	if (vm->reserved_vmid[vmhub]) {
-		list_add(&vm->reserved_vmid[vmhub]->list,
-			&id_mgr->ids_lru);
+	id = vm->reserved_vmid[vmhub];
+	if (id) {
 		vm->reserved_vmid[vmhub] = NULL;
 		id_mgr->reserved_vmid = false;
+
+		/* Only for kernel queues and kfd/userqueues by pass that */
+		amdgpu_sync_wait(&id->active, false);
+
+		if (adev->vm_manager.spm_vmid && adev->gfxhub.funcs)
+			amdgpu_gmc_bind_vmid(adev, AMDGPU_GFXHUB(0),
+					     adev->vm_manager.spm_vmid, 0);
 	}
 	mutex_unlock(&id_mgr->lock);
 }
@@ -628,41 +702,74 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev)
 		for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID)
 			amdgpu_vmid_reset(adev, i, j);
 	}
+
+	/*
+	 * The SPM VMID is not part of the normal kernel pool, reset it
+	 * anyway in case it is used with kernel queues.
+	 */
+	if (adev->vm_manager.spm_vmid)
+		for (i = AMDGPU_GFXHUB_START; i < AMDGPU_MMHUB0_START; ++i)
+			amdgpu_vmid_reset(adev, i, adev->vm_manager.spm_vmid);
 }
 
+/**
+ * amdgpu_vmid_mgr_set_vmid_mask - set the kernel- and user-queue VMID
+ * pools for both hubs
+ *
+ * @adev: amdgpu device structure
+ * @vmid_kq_mask_gfxhub: kernel-queue VMID pool mask for GFXHUB
+ * @vmid_kq_mask_mmhub: kernel-queue VMID pool mask for MMHUB
+ *
+ * Sets both GFXHUB and MMHUB kernel-queue VMID pools
+ * via id_mgr[].ids_lru, and KFD/MES user-queue masks too.
+ */
 void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev,
-				   unsigned long vmid_mask, bool for_mmhub)
+				   unsigned long vmid_kq_mask_gfxhub,
+				   unsigned long vmid_kq_mask_mmhub)
 {
+	unsigned long uq_mask_gfxhub, uq_mask_mmhub;
 	unsigned int i;
 
 	BUILD_BUG_ON(AMDGPU_NUM_VMID > BITS_PER_LONG);
 
+	/*
+	 * Derive the uq masks from the kq masks as given, then carve each
+	 * hub's dedicated out-of-pool VMID out of both of that hub's pools.
+	 */
+	uq_mask_gfxhub = GENMASK(AMDGPU_NUM_VMID - 1, 1) & ~vmid_kq_mask_gfxhub;
+	uq_mask_mmhub = 0xFF00;
+
+	/*
+	 * SPM uses a dedicated vmid that is reserved from gfxhub.
+	 * Unmask the reserved vmids from both user and kernel queues
+	 */
+	if (adev->vm_manager.spm_vmid) {
+		vmid_kq_mask_gfxhub &= ~BIT(adev->vm_manager.spm_vmid);
+		uq_mask_gfxhub &= ~BIT(adev->vm_manager.spm_vmid);
+	}
+
+	/*
+	 * NPA uses a dedicated vmid that is reserved from mmhub.
+	 * Unmask the reserved VMIDs from both user and kernel queues
+	 */
+	if (adev->vm_manager.npa_vmid) {
+		vmid_kq_mask_mmhub &= ~BIT(adev->vm_manager.npa_vmid);
+		uq_mask_mmhub &= ~BIT(adev->vm_manager.npa_vmid);
+	}
+
+	adev->vm_manager.vmid_uq_mask_gfxhub = uq_mask_gfxhub;
+	adev->vm_manager.vmid_uq_mask_mmhub = uq_mask_mmhub;
+
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) {
 		bool is_mmhub = AMDGPU_IS_MMHUB0(i) || AMDGPU_IS_MMHUB1(i);
 
-		if (is_mmhub == for_mmhub)
-			bitmap_copy(adev->vm_manager.id_mgr[i].vmid_mask,
-				    &vmid_mask, AMDGPU_NUM_VMID);
+		bitmap_copy(adev->vm_manager.id_mgr[i].vmid_mask,
+			    is_mmhub ? &vmid_kq_mask_mmhub : &vmid_kq_mask_gfxhub,
+			    AMDGPU_NUM_VMID);
 	}
 }
 
 /**
- * amdgpu_vmid_uq_mask_init - init the user queue VMID masks
- *
- * @adev: amdgpu_device pointer
- *
- * Sets vmid_uq_mask_gfxhub/mmhub to the pool of VMIDs at or above
- * first_kfd_vmid, shared by KFD and MES user queues.
- */
-static void amdgpu_vmid_uq_mask_init(struct amdgpu_device *adev)
-{
-	adev->vm_manager.vmid_uq_mask_gfxhub =
-		((1U << AMDGPU_NUM_VMID) - 1) &
-		~((1U << adev->vm_manager.first_kfd_vmid) - 1);
-	adev->vm_manager.vmid_uq_mask_mmhub = 0xFF00;
-}
-
-/**
  * amdgpu_vmid_mgr_init - init the VMID manager
  *
  * @adev: amdgpu_device pointer
@@ -673,8 +780,6 @@ void amdgpu_vmid_mgr_init(struct amdgpu_device *adev)
 {
 	unsigned i, j;
 
-	amdgpu_vmid_uq_mask_init(adev);
-
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
 		struct amdgpu_kq_vmid_mgr *id_mgr =
 			&adev->vm_manager.id_mgr[i];
@@ -689,6 +794,17 @@ void amdgpu_vmid_mgr_init(struct amdgpu_device *adev)
 			list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
 		}
 	}
+
+	/*
+	 * The SPM VMID is not part of the normal kernel pool, create its
+	 * sync anyway in case it is used with kernel queues.
+	 */
+	if (adev->vm_manager.spm_vmid) {
+		unsigned int spm_vmid = adev->vm_manager.spm_vmid;
+
+		for (i = AMDGPU_GFXHUB_START; i < AMDGPU_MMHUB0_START; ++i)
+			amdgpu_sync_create(&adev->vm_manager.id_mgr[i].ids[spm_vmid].active);
+	}
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
index 21d6a41d1738..c134000eecfd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
@@ -56,6 +56,22 @@
 	     __vmid_mask; __vmid_mask = 0)				\
 		for_each_set_bit((vmid), &__vmid_mask, AMDGPU_NUM_VMID)
 
+/**
+ * for_each_uq_gfx_vmid - iterate over every VMID set in vmid_uq_mask_gfxhub
+ * @vmid:	loop cursor (int / unsigned int)
+ * @adev:	struct amdgpu_device *
+ *
+ * Equivalent to walking adev->vm_manager.vmid_uq_mask_gfxhub with
+ * for_each_set_bit().
+ *
+ * Relies on AMDGPU_NUM_VMID (16) fitting in a single unsigned long.
+ */
+#define for_each_uq_gfx_vmid(vmid, adev)					\
+	for (unsigned long __uq_vmid_mask =				\
+		(adev)->vm_manager.vmid_uq_mask_gfxhub;		\
+	     __uq_vmid_mask; __uq_vmid_mask = 0)			\
+		for_each_set_bit((vmid), &__uq_vmid_mask, AMDGPU_NUM_VMID)
+
 struct amdgpu_device;
 struct amdgpu_fpriv;
 struct amdgpu_vm;
@@ -98,6 +114,9 @@ int amdgpu_pasid_alloc(unsigned int bits, struct amdgpu_fpriv *fpriv);
 void amdgpu_pasid_lock(unsigned long *flags);
 void amdgpu_pasid_unlock(unsigned long flags);
 struct amdgpu_fpriv *amdgpu_pasid_get_fpriv_locked(u32 pasid);
+int amdgpu_pasid_drain_irq(struct amdgpu_device *adev, u32 pasid,
+			   u32 *ih_fence, unsigned long timeout);
+void amdgpu_pasid_drain_irq_done(u32 pasid);
 void amdgpu_pasid_free(u32 pasid);
 void amdgpu_pasid_free_delayed(struct dma_resv *resv,
 			       u32 pasid);
@@ -106,9 +125,9 @@ void amdgpu_pasid_mgr_cleanup(void);
 bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev,
 			       struct amdgpu_vmid *id);
 bool amdgpu_vmid_uses_reserved(struct amdgpu_vm *vm, unsigned int vmhub);
-int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+int amdgpu_vmid_alloc_spm(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			       unsigned vmhub);
-void amdgpu_vmid_free_reserved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+void amdgpu_vmid_free_spm(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			       unsigned vmhub);
 int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 		     struct amdgpu_job *job, struct dma_fence **fence);
@@ -119,6 +138,7 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev);
 void amdgpu_vmid_mgr_init(struct amdgpu_device *adev);
 void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev);
 void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev,
-				   unsigned long vmid_mask, bool for_mmhub);
+				   unsigned long vmid_kq_mask_gfxhub,
+				   unsigned long vmid_kq_mask_mmhub);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index 57bd34981be0..18e788f0e790 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -75,7 +75,6 @@ struct amdgpu_job {
 	uint32_t                preemption_status;
 	bool                    vm_needs_flush;
 	bool			gds_switch_needed;
-	bool			spm_update_needed;
 	uint64_t		vm_pd_addr;
 	unsigned		vmid;
 	unsigned		pasid;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 8e15b204f117..7c9c58af0658 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -780,9 +780,16 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 
 		return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
 	}
-	case AMDGPU_INFO_TIMESTAMP:
+	case AMDGPU_INFO_TIMESTAMP: {
+		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
+
+		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+		if (ret)
+			return ret;
+
 		ui64 = amdgpu_gfx_get_gpu_clock_counter(adev);
 		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
+	}
 	case AMDGPU_INFO_FW_VERSION: {
 		struct drm_amdgpu_info_firmware fw_info;
 
@@ -906,6 +913,12 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		if (info->read_mmr_reg.count > 128)
 			return -EINVAL;
 
+		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
+
+		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+		if (ret)
+			return ret;
+
 		regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs),
 				     GFP_KERNEL);
 		if (!regs)
@@ -940,6 +953,13 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		uint64_t vm_size;
 		uint32_t pcie_gen_mask, pcie_width_mask;
 
+		/* The clock limits are only available while the SMU is up. */
+		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
+
+		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+		if (ret)
+			return ret;
+
 		dev_info = kzalloc_obj(*dev_info);
 		if (!dev_info)
 			return -ENOMEM;
@@ -1167,6 +1187,12 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		}
 	}
 	case AMDGPU_INFO_SENSOR: {
+		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
+
+		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+		if (ret)
+			return ret;
+
 		if (!adev->pm.dpm_enabled)
 			return -ENOENT;
 
@@ -1405,24 +1431,24 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 				return ret;
 
 			ret = copy_to_user(out, &meta_info,
-						min((size_t)size, sizeof(meta_info))) ? -EFAULT : 0;
-			return 0;
+						min((size_t)size, sizeof(meta_info)));
+			return ret ? -EFAULT : 0;
 		case AMDGPU_HW_IP_COMPUTE:
 			ret = amdgpu_userq_metadata_info_compute(adev, info, &meta_info.compute);
 			if (ret)
 				return ret;
 
 			ret = copy_to_user(out, &meta_info,
-						min((size_t)size, sizeof(meta_info))) ? -EFAULT : 0;
-			return 0;
+						min((size_t)size, sizeof(meta_info)));
+			return ret ? -EFAULT : 0;
 		case AMDGPU_HW_IP_DMA:
 			ret = amdgpu_userq_metadata_info_sdma(adev, info, &meta_info.sdma);
 			if (ret)
 				return ret;
 
 			ret = copy_to_user(out, &meta_info,
-						min((size_t)size, sizeof(meta_info))) ? -EFAULT : 0;
-			return 0;
+						min((size_t)size, sizeof(meta_info)));
+			return ret ? -EFAULT : 0;
 		default:
 			return -EINVAL;
 		}
@@ -1508,6 +1534,8 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	if (r)
 		goto error_pasid;
 
+	init_completion(&fpriv->irq_drain_done);
+
 	pasid = amdgpu_pasid_alloc(16, fpriv);
 	if (pasid < 0) {
 		dev_warn(adev->dev, "No more PASIDs available!");
@@ -1643,6 +1671,7 @@ void amdgpu_driver_release_kms(struct drm_device *dev)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
 
+	amdgpu_xcp_dev_fini(adev);
 	amdgpu_device_fini_sw(adev);
 	pci_set_drvdata(adev->pdev, NULL);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 8d813c68450b..2ff1816adfc4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -592,6 +592,7 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg,
 	op_input.op = MES_MISC_OP_READ_REG;
 	op_input.read_reg.reg_offset = reg;
 	op_input.read_reg.buffer_addr = read_val_gpu_addr;
+	op_input.read_reg.read64_bits = false;
 
 	if (!adev->mes.funcs->misc_op) {
 		dev_err(adev->dev, "mes rreg is not supported!\n");
@@ -612,6 +613,46 @@ error:
 	return val;
 }
 
+int amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg,
+		      uint32_t xcc_id, uint64_t *val)
+{
+	struct mes_misc_op_input op_input = { 0 };
+	uint32_t addr_offset;
+	uint64_t *read_val_ptr;
+	int r;
+
+	r = amdgpu_wb_get(adev, &addr_offset);
+	if (r) {
+		dev_err(adev->dev, "critical bug! too many mes readers\n");
+		return r;
+	}
+
+	read_val_ptr = (uint64_t *)&adev->wb.wb[addr_offset];
+	op_input.xcc_id = xcc_id;
+	op_input.op = MES_MISC_OP_READ_REG;
+	op_input.read_reg.reg_offset = reg;
+	op_input.read_reg.buffer_addr = adev->wb.gpu_addr + addr_offset * 4;
+	op_input.read_reg.read64_bits = true;
+
+	if (!adev->mes.funcs->misc_op) {
+		dev_err(adev->dev, "mes rreg is not supported!\n");
+		r = -EINVAL;
+		goto error;
+	}
+
+	amdgpu_mes_lock(&adev->mes);
+	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
+	amdgpu_mes_unlock(&adev->mes);
+	if (r)
+		dev_err(adev->dev, "failed to read reg (0x%x)\n", reg);
+	else
+		*val = READ_ONCE(*read_val_ptr);
+
+error:
+	amdgpu_wb_free(adev, addr_offset);
+	return r;
+}
+
 int amdgpu_mes_wreg(struct amdgpu_device *adev, uint32_t reg,
 		    uint32_t val, uint32_t xcc_id)
 {
@@ -695,6 +736,7 @@ int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
 				uint64_t process_context_addr,
 				uint32_t spi_gdbg_per_vmid_cntl,
 				const uint32_t *tcp_watch_cntl,
+				const uint64_t *watch_addresses,
 				uint32_t flags,
 				bool trap_en,
 				uint32_t xcc_id)
@@ -720,6 +762,8 @@ int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
 	op_input.set_shader_debugger.spi_gdbg_per_vmid_cntl = spi_gdbg_per_vmid_cntl;
 	memcpy(op_input.set_shader_debugger.tcp_watch_cntl, tcp_watch_cntl,
 			sizeof(op_input.set_shader_debugger.tcp_watch_cntl));
+	memcpy(op_input.set_shader_debugger.watch_addresses, watch_addresses,
+			sizeof(op_input.set_shader_debugger.watch_addresses));
 
 	if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>
 			AMDGPU_MES_API_VERSION_SHIFT) >= 14)
@@ -1339,8 +1383,13 @@ static void mes_dbgext_buf_read(const u8 *buf, u32 buffer_size, u32 hdr_size,
 	}
 }
 
+static const char *mes_dbgext_pipe_name(u32 pipe)
+{
+	return pipe == AMDGPU_MES_SCHED_PIPE ? "0" : "1";
+}
+
 static void mes_dbgext_print_item(struct amdgpu_device *adev, int xcc,
-				  u32 type, char *text)
+				  int pipe, u32 type, char *text)
 {
 	size_t n = strlen(text);
 	char pfx[12] = "";
@@ -1351,21 +1400,24 @@ static void mes_dbgext_print_item(struct amdgpu_device *adev, int xcc,
 	while (n && (text[n - 1] == '\n' || text[n - 1] == '\r'))
 		text[--n] = '\0';
 
-	/* Tag with the source XCC only when more than one is being logged, so
-	 * single-XCC (gfx11/gfx12) output is unchanged.
+	/* Tag with the source XCC and MES pipe (sched/kiq) only when more than
+	 * one region is being logged, so single-region output is unchanged.
 	 */
-	if (adev->mes.dbgext_num_xcc > 1)
-		snprintf(pfx, sizeof(pfx), " xcc%d", xcc);
+	if (adev->mes.dbgext_num_pipes > 1)
+		snprintf(pfx, sizeof(pfx), "(%d,%s)", xcc,
+			 mes_dbgext_pipe_name(pipe));
+	else if (adev->mes.dbgext_num_xcc > 1)
+		snprintf(pfx, sizeof(pfx), "(%d)", xcc);
 
 	switch (type) {
 	case MES_DBGEXT_MSG_ASSERT:
-		dev_err(adev->dev, "[mes_dbgext%s] ASSERT %s\n", pfx, text);
+		dev_err(adev->dev, "[mes_dbgext] %s ASSERT %s\n", pfx, text);
 		break;
 	case MES_DBGEXT_MSG:
-		dev_info(adev->dev, "[mes_dbgext%s] %s\n", pfx, text);
+		dev_info(adev->dev, "[mes_dbgext] %s %s\n", pfx, text);
 		break;
 	default:
-		dev_warn(adev->dev, "[mes_dbgext%s] %s\n", pfx, text);
+		dev_warn(adev->dev, "[mes_dbgext] %s %s\n", pfx, text);
 		break;
 	}
 }
@@ -1376,7 +1428,7 @@ static void mes_dbgext_print_item(struct amdgpu_device *adev, int xcc,
  * caller-provided scratch of at least MES_DBGEXT_MAX_ITEM_SIZE + 1 bytes.
  */
 static void mes_dbgext_process_zone(struct amdgpu_device *adev, u8 *zbase,
-				    u8 *item, int xcc)
+				    u8 *item, int xcc, int pipe)
 {
 	struct mes_dbgext_zone_header *zh =
 		(struct mes_dbgext_zone_header *)zbase;
@@ -1438,7 +1490,7 @@ static void mes_dbgext_process_zone(struct amdgpu_device *adev, u8 *zbase,
 			mes_dbgext_buf_read(zbase, buffer_size, hdr_size, rptr,
 					    item, len);
 			item[len] = '\0';
-			mes_dbgext_print_item(adev, xcc, type,
+			mes_dbgext_print_item(adev, xcc, pipe, type,
 				(char *)item + MES_DBGEXT_ITEM_HDR_SIZE);
 		}
 
@@ -1461,11 +1513,12 @@ static void mes_dbgext_process_zone(struct amdgpu_device *adev, u8 *zbase,
 }
 
 /*
- * Drain one per-XCC region: parse its zone table (a zone_count dword followed
- * by per-zone {offset,length} descriptors) and drain each zone.  @base points
- * at the region start; @xcc tags the output.
+ * Drain one per-(xcc, pipe) region: parse its zone table (a zone_count dword
+ * followed by per-zone {offset,length} descriptors) and drain each zone.
+ * @base points at the region start; @xcc and @pipe tag the output.
  */
-static void mes_dbgext_process_region(struct amdgpu_device *adev, u8 *base, int xcc)
+static void mes_dbgext_process_region(struct amdgpu_device *adev, u8 *base,
+				      int xcc, int pipe)
 {
 	struct amdgpu_mes *mes = &adev->mes;
 	u32 zone_count, hdr_len, z;
@@ -1499,32 +1552,41 @@ static void mes_dbgext_process_region(struct amdgpu_device *adev, u8 *base, int
 		    zlen > mes->dbgext_log_size - zoff)
 			continue;
 
-		mes_dbgext_process_zone(adev, base + zoff, item, xcc);
+		mes_dbgext_process_zone(adev, base + zoff, item, xcc, pipe);
 	}
 
 	kfree(item);
 }
 
+static u32 mes_dbgext_num_pipes(struct amdgpu_mes *mes)
+{
+	return mes->dbgext_num_pipes ? mes->dbgext_num_pipes : 1;
+}
+
+static u32 mes_dbgext_num_regions(struct amdgpu_mes *mes)
+{
+	return (mes->dbgext_num_xcc ? mes->dbgext_num_xcc : 1) *
+	       mes_dbgext_num_pipes(mes);
+}
+
 static void mes_dbgext_process_all(struct amdgpu_device *adev)
 {
 	struct amdgpu_mes *mes = &adev->mes;
-	u32 num_xcc = mes->dbgext_num_xcc ? mes->dbgext_num_xcc : 1;
+	u32 num_pipes = mes_dbgext_num_pipes(mes);
+	u32 num_regions = mes_dbgext_num_regions(mes);
 	u8 *buf = mes->dbgext_log_cpu_addr;
-	u32 xcc;
+	u32 i;
 
 	if (!buf)
 		return;
 
 	/*
-	 * The buffer holds num_xcc back-to-back per-XCC regions, each
-	 * dbgext_log_size bytes.  Drain each region and tag its output with the
-	 * source XCC (single-XCC ASICs have exactly one region).
+	 * Every armed (xcc, pipe) raises the same host interrupt, so drain all
+	 * regions; each is a no-op when it has no new data.
 	 */
-	for (xcc = 0; xcc < num_xcc; xcc++) {
-		u8 *base = buf + xcc * mes->dbgext_log_size;
-
-		mes_dbgext_process_region(adev, base, xcc);
-	}
+	for (i = 0; i < num_regions; i++)
+		mes_dbgext_process_region(adev, buf + i * mes->dbgext_log_size,
+					  i / num_pipes, i % num_pipes);
 }
 
 static int amdgpu_mes_dbgext_reader(void *param)
@@ -1577,7 +1639,8 @@ void amdgpu_mes_dbgext_notify(struct amdgpu_device *adev, u32 context_data)
 }
 
 static int amdgpu_mes_dbgext_setup_fw(struct amdgpu_device *adev, u32 xcc_id,
-				      u64 log_buffer_mc_addr, u64 log_options)
+				      u32 pipe, u64 log_buffer_mc_addr,
+				      u64 log_options)
 {
 	struct amdgpu_mes *mes = &adev->mes;
 	struct mes_misc_op_input op_input = {0};
@@ -1590,6 +1653,7 @@ static int amdgpu_mes_dbgext_setup_fw(struct amdgpu_device *adev, u32 xcc_id,
 	op_input.op = MES_MISC_OP_SETUP_MES_DBGEXT;
 	op_input.setup_mes_dbgext.log_buffer_mc_addr = log_buffer_mc_addr;
 	op_input.setup_mes_dbgext.log_options = log_options;
+	op_input.setup_mes_dbgext.pipe = pipe;
 
 	amdgpu_mes_lock(mes);
 	r = mes->funcs->misc_op(mes, &op_input);
@@ -1606,7 +1670,7 @@ static int amdgpu_mes_dbgext_start_locked(struct amdgpu_device *adev)
 	u64 fw_options;
 	u32 size;
 	u32 req_kb;
-	u32 num_xcc, xcc;
+	u32 num_xcc, num_pipes, num_regions, i;
 	int r;
 
 	/*
@@ -1636,15 +1700,21 @@ static int amdgpu_mes_dbgext_start_locked(struct amdgpu_device *adev)
 	}
 
 	/*
-	 * Log every XCC's MES firmware.  Each XCC runs its own MES and gets its
-	 * own per-XCC region within one buffer (see dbgext_num_xcc); single-XCC
-	 * ASICs (gfx11/gfx12) collapse to a single region.
+	 * Log every MES firmware instance: each XCC runs its own MES, and under
+	 * unified MES both pipes of each XCC run firmware with their own
+	 * context, so each (xcc, pipe) gets its own region within one buffer
+	 * (see dbgext_num_xcc/dbgext_num_pipes).  Without unified MES the KIQ
+	 * pipe is driven through the gfx KIQ ring and cannot take MES API
+	 * packets, so only the scheduler pipe is logged.
 	 */
 	num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
+	num_pipes = adev->enable_uni_mes ? AMDGPU_MAX_MES_PIPES : 1;
+	num_regions = num_xcc * num_pipes;
 	mes->dbgext_num_xcc = num_xcc;
+	mes->dbgext_num_pipes = num_pipes;
 
 	if (!mes->dbgext_log_gpu_obj) {
-		/* Clamp to a sane range (4 KB .. 1 MB), per XCC. */
+		/* Clamp to a sane range (4 KB .. 1 MB), per region. */
 		size = clamp(req_kb, 4U, 1024U);
 		size = ALIGN((u32)size * SZ_1K, PAGE_SIZE);
 		/*
@@ -1655,8 +1725,8 @@ static int amdgpu_mes_dbgext_start_locked(struct amdgpu_device *adev)
 		if (size < SZ_8K)
 			size = SZ_8K;
 
-		/* One buffer holds num_xcc back-to-back per-XCC regions. */
-		r = amdgpu_bo_create_kernel(adev, size * num_xcc, PAGE_SIZE,
+		/* One buffer holds num_regions back-to-back regions. */
+		r = amdgpu_bo_create_kernel(adev, size * num_regions, PAGE_SIZE,
 					    AMDGPU_GEM_DOMAIN_GTT,
 					    &mes->dbgext_log_gpu_obj,
 					    &mes->dbgext_log_gpu_addr,
@@ -1678,13 +1748,13 @@ static int amdgpu_mes_dbgext_start_locked(struct amdgpu_device *adev)
 	}
 
 	/*
-	 * (Re)initialize each per-XCC region's header.  On resume the MES was
-	 * reset and re-lays its header, and the driver's rptr/wptr must restart
-	 * clean.  Clear the whole buffer once, then seed every region.
+	 * (Re)initialize each region's header.  On resume the MES was reset and
+	 * re-lays its header, and the driver's rptr/wptr must restart clean.
+	 * Clear the whole buffer once, then seed every region.
 	 */
-	memset(mes->dbgext_log_cpu_addr, 0, size * num_xcc);
-	for (xcc = 0; xcc < num_xcc; xcc++) {
-		u8 *base = (u8 *)mes->dbgext_log_cpu_addr + xcc * size;
+	memset(mes->dbgext_log_cpu_addr, 0, size * num_regions);
+	for (i = 0; i < num_regions; i++) {
+		u8 *base = (u8 *)mes->dbgext_log_cpu_addr + i * size;
 
 		/*
 		 * mes_aux: seed only the total region size in the first dword.
@@ -1736,18 +1806,20 @@ static int amdgpu_mes_dbgext_start_locked(struct amdgpu_device *adev)
 	if (use_irq)
 		mes->funcs->enable_dbgext_irq(mes, true);
 
-	/* Point each XCC's MES firmware at its own per-XCC region. */
-	for (xcc = 0; xcc < num_xcc; xcc++) {
-		r = amdgpu_mes_dbgext_setup_fw(adev, xcc,
-					       mes->dbgext_log_gpu_addr + xcc * size,
+	/* Point each (xcc, pipe) MES firmware at its own region. */
+	for (i = 0; i < num_regions; i++) {
+		r = amdgpu_mes_dbgext_setup_fw(adev, i / num_pipes, i % num_pipes,
+					       mes->dbgext_log_gpu_addr + i * size,
 					       fw_options);
 		if (r) {
 			dev_err(adev->dev,
-				"failed to setup mes_dbgext in FW on xcc%u (%d)\n",
-				xcc, r);
-			/* Detach the XCCs already armed before bailing. */
-			while (xcc--)
-				amdgpu_mes_dbgext_setup_fw(adev, xcc, 0, 0);
+				"failed to setup mes_dbgext in FW on xcc%u %s pipe (%d)\n",
+				i / num_pipes,
+				mes_dbgext_pipe_name(i % num_pipes), r);
+			/* Detach the regions already armed before bailing. */
+			while (i--)
+				amdgpu_mes_dbgext_setup_fw(adev, i / num_pipes,
+							   i % num_pipes, 0, 0);
 			goto err_disable;
 		}
 	}
@@ -1775,8 +1847,8 @@ static int amdgpu_mes_dbgext_start_locked(struct amdgpu_device *adev)
 	}
 
 	dev_info(adev->dev,
-		 "mes_dbgext enabled: %u KB x %u xcc @ 0x%llx (%s, fw options 0x%llx)\n",
-		 size / SZ_1K, num_xcc, mes->dbgext_log_gpu_addr,
+		 "mes_dbgext enabled: %u KB x %u xcc x %u pipe @ 0x%llx (%s, fw options 0x%llx)\n",
+		 size / SZ_1K, num_xcc, num_pipes, mes->dbgext_log_gpu_addr,
 		 use_irq ? "interrupt" : "polling", fw_options);
 
 	/*
@@ -1792,8 +1864,9 @@ static int amdgpu_mes_dbgext_start_locked(struct amdgpu_device *adev)
 	return 0;
 
 err_fw_detach:
-	for (xcc = 0; xcc < num_xcc; xcc++)
-		amdgpu_mes_dbgext_setup_fw(adev, xcc, 0, 0);
+	for (i = 0; i < num_regions; i++)
+		amdgpu_mes_dbgext_setup_fw(adev, i / num_pipes, i % num_pipes,
+					   0, 0);
 err_disable:
 	/* Undo the early arming done before setup_fw(). */
 	mes->dbgext_active = false;
@@ -1850,17 +1923,19 @@ static void amdgpu_mes_dbgext_stop_locked(struct amdgpu_device *adev)
 		return;
 
 	/*
-	 * Detach the firmware from the buffer before we free it, on every XCC we
-	 * armed.  Skip this across a suspend: the MES is being torn down anyway,
-	 * the buffer is kept for resume, and submitting a packet on the suspend
-	 * path is unnecessary.
+	 * Detach the firmware from the buffer before we free it, on every
+	 * (xcc, pipe) we armed.  Skip this across a suspend: the MES is being
+	 * torn down anyway, the buffer is kept for resume, and submitting a
+	 * packet on the suspend path is unnecessary.
 	 */
 	if (!adev->in_suspend) {
-		u32 num_xcc = mes->dbgext_num_xcc ? mes->dbgext_num_xcc : 1;
-		u32 xcc;
+		u32 num_pipes = mes_dbgext_num_pipes(mes);
+		u32 num_regions = mes_dbgext_num_regions(mes);
+		u32 i;
 
-		for (xcc = 0; xcc < num_xcc; xcc++)
-			amdgpu_mes_dbgext_setup_fw(adev, xcc, 0, 0);
+		for (i = 0; i < num_regions; i++)
+			amdgpu_mes_dbgext_setup_fw(adev, i / num_pipes,
+						   i % num_pipes, 0, 0);
 	}
 
 	/* Make sure no drain work is still touching the buffer. */
@@ -1922,8 +1997,10 @@ static int amdgpu_debugfs_mes_dbgext_show(struct seq_file *m, void *unused)
 
 	mutex_lock(&mes->dbgext_lock);
 	seq_printf(m, "state:   %s\n", mes->dbgext_active ? "on" : "off");
-	seq_printf(m, "size:    %u KB x %u xcc\n", mes->dbgext_log_size / SZ_1K,
-		   mes->dbgext_num_xcc ? mes->dbgext_num_xcc : 1);
+	seq_printf(m, "size:    %u KB x %u xcc x %u pipe\n",
+		   mes->dbgext_log_size / SZ_1K,
+		   mes->dbgext_num_xcc ? mes->dbgext_num_xcc : 1,
+		   mes_dbgext_num_pipes(mes));
 	seq_printf(m, "mode:    %s\n",
 		   mes->dbgext_use_irq ? "interrupt" : "polling");
 	seq_printf(m, "options: 0x%llx\n", mes->dbgext_log_options);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 2b13a5b37539..0f368fbd2c80 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -154,7 +154,7 @@ struct amdgpu_mes {
 	void				*event_log_cpu_addr;
 
 	/* MES firmware debug extension ("mes_dbgext") log buffer */
-	uint32_t			dbgext_log_size;	/* per-XCC region size */
+	uint32_t			dbgext_log_size;	/* per-region size */
 	uint64_t			dbgext_log_options;
 	struct amdgpu_bo		*dbgext_log_gpu_obj;
 	uint64_t			dbgext_log_gpu_addr;
@@ -162,10 +162,13 @@ struct amdgpu_mes {
 	bool				dbgext_use_irq;
 
 	/*
-	 * Number of per-XCC log regions packed back-to-back in the buffer above;
-	 * each is dbgext_log_size bytes.  1 on single-XCC ASICs (gfx11/gfx12).
+	 * The buffer above holds dbgext_num_xcc * dbgext_num_pipes back-to-back
+	 * log regions of dbgext_log_size bytes, one per (xcc, MES pipe), at
+	 * index xcc * dbgext_num_pipes + pipe.  dbgext_num_pipes is 2 under
+	 * unified MES (scheduler + KIQ pipe each run firmware), else 1.
 	 */
 	uint32_t			dbgext_num_xcc;
+	uint32_t			dbgext_num_pipes;
 
 	/* armed: FW attached + reader/irq running */
 	bool					dbgext_active;
@@ -405,6 +408,7 @@ struct mes_misc_op_input {
 		struct {
 			uint32_t                  reg_offset;
 			uint64_t                  buffer_addr;
+			bool                      read64_bits;
 		} read_reg;
 
 		struct {
@@ -432,6 +436,7 @@ struct mes_misc_op_input {
 			} flags;
 			uint32_t spi_gdbg_per_vmid_cntl;
 			uint32_t tcp_watch_cntl[4];
+			uint64_t watch_addresses[4];
 			uint32_t trap_en;
 		} set_shader_debugger;
 
@@ -453,6 +458,7 @@ struct mes_misc_op_input {
 		struct {
 			uint64_t log_buffer_mc_addr;
 			uint64_t log_options;
+			uint32_t pipe;
 		} setup_mes_dbgext;
 	};
 };
@@ -609,6 +615,8 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 
 uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg,
 			 uint32_t xcc_id);
+int amdgpu_mes_rreg64(struct amdgpu_device *adev, uint32_t reg,
+		      uint32_t xcc_id, uint64_t *val);
 int amdgpu_mes_wreg(struct amdgpu_device *adev,
 		    uint32_t reg, uint32_t val, uint32_t xcc_id);
 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
@@ -619,6 +627,7 @@ int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
 				uint64_t process_context_addr,
 				uint32_t spi_gdbg_per_vmid_cntl,
 				const uint32_t *tcp_watch_cntl,
+				const uint64_t *watch_addresses,
 				uint32_t flags,
 				bool trap_en,
 				uint32_t xcc_id);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
index e4c8e98725f0..55049a8635bd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
@@ -21,6 +21,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ras.h"
+#include "soc15_common.h"
 
 int amdgpu_nbio_ras_sw_init(struct amdgpu_device *adev)
 {
@@ -94,3 +95,152 @@ void amdgpu_nbio_program_aspm(struct amdgpu_device *adev)
 	if (adev->nbio.funcs->program_aspm)
 		adev->nbio.funcs->program_aspm(adev);
 }
+
+static int amdgpu_nbio_set_doorbell_field(u32 *reg, u32 mask, u32 value)
+{
+	u32 shift;
+
+	if (!mask)
+		return value ? -EOPNOTSUPP : 0;
+
+	shift = __ffs(mask);
+	if (value > (mask >> shift))
+		return -ERANGE;
+
+	*reg |= (value << shift) & mask;
+	return 0;
+}
+
+int amdgpu_nbio_build_doorbell_entry(
+	struct amdgpu_device *adev,
+	const struct amdgpu_nbio_doorbell_fields *fields,
+	u32 *ctrl, u32 *ctrl1)
+{
+	const struct amdgpu_nbio_doorbell_layout *layout;
+	int r;
+
+	if (!fields || !ctrl)
+		return -EINVAL;
+
+	layout = adev->nbio.doorbell_layout;
+	if (!layout)
+		return -EINVAL;
+
+	*ctrl = 0;
+	if (ctrl1)
+		*ctrl1 = 0;
+
+	r = amdgpu_nbio_set_doorbell_field(ctrl, layout->ctrl.enable,
+					   fields->enable);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl, layout->ctrl.awid,
+					   fields->awid);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl, layout->ctrl.fence_enable,
+					   fields->fence_enable);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl, layout->ctrl.range_offset,
+					   fields->range_offset);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl, layout->ctrl.range_size,
+					   fields->range_size);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl, layout->ctrl.disable_64bit,
+					   fields->disable_64bit);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl,
+					   layout->ctrl.deduct_range_offset,
+					   fields->deduct_range_offset);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl, layout->ctrl.drop,
+					   fields->drop);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl, layout->ctrl.awaddr_31_28,
+					   fields->awaddr_31_28);
+	if (r)
+		return r;
+
+	if (!ctrl1)
+		return 0;
+	if (!layout->has_ctrl1)
+		return -EOPNOTSUPP;
+
+	r = amdgpu_nbio_set_doorbell_field(ctrl1,
+					   layout->ctrl1.target_port_type,
+					   fields->target_port_type);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl1,
+					   layout->ctrl1.target_die_id,
+					   fields->target_die_id);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl1,
+					   layout->ctrl1.target_port_id,
+					   fields->target_port_id);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl1,
+					   layout->ctrl1.override_poison_data,
+					   fields->override_poison_data);
+	if (r)
+		return r;
+	r = amdgpu_nbio_set_doorbell_field(ctrl1,
+					   layout->ctrl1.disable_4k_page_decode,
+					   fields->disable_4k_page_decode);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+void amdgpu_nbio_program_doorbell_entry(
+	struct amdgpu_device *adev,
+	enum amdgpu_nbio_doorbell_client client,
+	u32 instance, const u32 *ctrl, const u32 *ctrl1)
+{
+	struct amdgpu_nbio_doorbell_entries *entries;
+	struct amdgpu_nbio_doorbell_entry *entry;
+
+	if (WARN_ON_ONCE(client >= AMDGPU_NBIO_DOORBELL_CLIENT_COUNT))
+		return;
+
+	entries = &adev->nbio.doorbell_entries[client];
+	if (WARN_ON_ONCE(instance >= entries->num_entries))
+		return;
+
+	entry = &entries->entry[instance];
+	if (WARN_ON_ONCE(!entry->ctrl_reg))
+		return;
+
+	if (entry->use_ext) {
+		/* CTRL1 programming is not needed when use_ext is set. */
+		if (WARN_ON_ONCE(entry->ctrl1_reg || ctrl1))
+			return;
+
+		WREG32_PCIE_EXT(
+			(adev->reg_offset[NBIO_HWIP][entry->reg_inst]
+			 [entry->ctrl_reg_base_idx] + entry->ctrl_reg) * 4 +
+			amdgpu_reg_get_smn_base64(
+				adev, NBIO_HWIP, entry->reg_inst),
+			ctrl ? *ctrl : 0);
+		return;
+	}
+
+	if (WARN_ON_ONCE(ctrl1 && !entry->ctrl1_reg))
+		return;
+
+	/* A NULL CTRL disables and clears the entry. */
+	WREG32_SOC15_IP(NBIO, entry->ctrl_reg, ctrl ? *ctrl : 0);
+	if (entry->ctrl1_reg && (!ctrl || ctrl1))
+		WREG32_SOC15_IP(NBIO, entry->ctrl1_reg,
+				    ctrl ? *ctrl1 : 0);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
index a61f3a6e8ec7..18418a331517 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
@@ -55,6 +55,82 @@ struct amdgpu_nbio_ras {
 	int (*init_ras_err_event_athub_interrupt)(struct amdgpu_device *adev);
 };
 
+enum amdgpu_nbio_doorbell_client {
+	AMDGPU_NBIO_DOORBELL_CLIENT_CP,
+	AMDGPU_NBIO_DOORBELL_CLIENT_IH,
+	AMDGPU_NBIO_DOORBELL_CLIENT_SDMA,
+	AMDGPU_NBIO_DOORBELL_CLIENT_RLC,
+	AMDGPU_NBIO_DOORBELL_CLIENT_VCN,
+	AMDGPU_NBIO_DOORBELL_CLIENT_VPE,
+	AMDGPU_NBIO_DOORBELL_CLIENT_COUNT,
+};
+
+struct amdgpu_nbio_doorbell_ctrl_layout {
+	u32 enable;
+	u32 awid;
+	u32 fence_enable;
+	u32 range_offset;
+	u32 range_size;
+	u32 disable_64bit;
+	u32 deduct_range_offset;
+	u32 drop;
+	u32 awaddr_31_28;
+};
+
+struct amdgpu_nbio_doorbell_ctrl1_layout {
+	u32 target_port_type;
+	u32 target_die_id;
+	u32 target_port_id;
+	u32 override_poison_data;
+	u32 disable_4k_page_decode;
+};
+
+struct amdgpu_nbio_doorbell_layout {
+	struct amdgpu_nbio_doorbell_ctrl_layout ctrl;
+	struct amdgpu_nbio_doorbell_ctrl1_layout ctrl1;
+	bool has_ctrl1;
+};
+
+struct amdgpu_nbio_doorbell_fields {
+	bool enable;
+	u32 awid;
+	u32 range_offset;
+	u32 range_size;
+	u32 awaddr_31_28;
+	u32 target_port_type;
+	u32 target_die_id;
+	u32 target_port_id;
+	bool fence_enable;
+	bool disable_64bit;
+	bool deduct_range_offset;
+	bool drop;
+	bool override_poison_data;
+	bool disable_4k_page_decode;
+};
+
+#define AMDGPU_NBIO_MAX_DOORBELL_INSTANCES 16
+
+struct amdgpu_nbio_doorbell_entry {
+	u32 port;
+	u32 ctrl_reg;
+	u32 ctrl1_reg;
+	/*
+	 * Some NBIO generations expose each register instance through a global
+	 * SMN address. The caller invokes the doorbell callback for each required
+	 * instance across the die; reg_inst identifies the instance represented
+	 * by this entry. Extended entries only support a CTRL register.
+	 */
+	bool use_ext;
+	u32 reg_inst;
+	u32 ctrl_reg_base_idx;
+};
+
+struct amdgpu_nbio_doorbell_entries {
+	u32 num_entries;
+	struct amdgpu_nbio_doorbell_entry
+		entry[AMDGPU_NBIO_MAX_DOORBELL_INSTANCES];
+};
+
 struct amdgpu_nbio_funcs {
 	const struct nbio_hdp_flush_reg *hdp_flush_reg;
 	u32 (*get_hdp_flush_req_offset)(struct amdgpu_device *adev);
@@ -113,6 +189,9 @@ struct amdgpu_nbio {
 	struct ras_common_if *ras_if;
 	const struct amdgpu_nbio_funcs *funcs;
 	struct amdgpu_nbio_ras  *ras;
+	const struct amdgpu_nbio_doorbell_layout *doorbell_layout;
+	struct amdgpu_nbio_doorbell_entries
+		doorbell_entries[AMDGPU_NBIO_DOORBELL_CLIENT_COUNT];
 };
 
 int amdgpu_nbio_ras_sw_init(struct amdgpu_device *adev);
@@ -123,4 +202,13 @@ bool amdgpu_nbio_is_replay_cnt_supported(struct amdgpu_device *adev);
 
 void amdgpu_nbio_program_aspm(struct amdgpu_device *adev);
 
+int amdgpu_nbio_build_doorbell_entry(
+	struct amdgpu_device *adev,
+	const struct amdgpu_nbio_doorbell_fields *fields,
+	u32 *ctrl, u32 *ctrl1);
+void amdgpu_nbio_program_doorbell_entry(
+	struct amdgpu_device *adev,
+	enum amdgpu_nbio_doorbell_client client,
+	u32 instance, const u32 *ctrl, const u32 *ctrl1);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 2a6a5bc01eb4..6aa895f63aea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -923,6 +923,19 @@ void amdgpu_bo_unref(struct amdgpu_bo **bo)
 	*bo = NULL;
 }
 
+/* Map a BO onto vram_pin_size_xcp[]. bo->xcp_id < 0 means "any partition":
+ * such pins are not owned by a single partition and go to
+ * vram_pin_size_common, which every partition subtracts.
+ */
+static int amdgpu_bo_get_xcp(struct amdgpu_bo *bo)
+{
+	s8 xcp = bo->xcp_id;
+
+	if (xcp >= 0 && xcp < MAX_XCP)
+		return xcp;
+	return -1;
+}
+
 /**
  * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
  * @bo: &amdgpu_bo buffer object to be pinned
@@ -1003,7 +1016,17 @@ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
 	ttm_bo_pin(&bo->tbo);
 
 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
+		int xcp = amdgpu_bo_get_xcp(bo);
+
 		atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
+		if (xcp >= 0) {
+			atomic64_add(amdgpu_bo_size(bo),
+				     &adev->vram_pin_size_xcp[xcp]);
+		} else {
+			atomic64_add(amdgpu_bo_size(bo),
+				     &adev->vram_pin_size_common);
+		}
+		amdgpu_amdkfd_account_vram_pin(bo, xcp);
 		atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
 			     &adev->visible_pin_size);
 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
@@ -1036,7 +1059,17 @@ void amdgpu_bo_unpin(struct amdgpu_bo *bo)
 		dma_buf_unpin(bo->tbo.base.import_attach);
 
 	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
+		int xcp = amdgpu_bo_get_xcp(bo);
+
 		atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
+		if (xcp >= 0) {
+			atomic64_sub(amdgpu_bo_size(bo),
+				     &adev->vram_pin_size_xcp[xcp]);
+		} else {
+			atomic64_sub(amdgpu_bo_size(bo),
+				     &adev->vram_pin_size_common);
+		}
+		amdgpu_amdkfd_account_vram_unpin(bo, xcp);
 		atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
 			     &adev->visible_pin_size);
 	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 82a2684dbd2e..36bc4b431dd5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -958,9 +958,13 @@ psp_cmd_submit_buf(struct psp_context *psp,
 				psp->cmd_resp_buf_mem->resp.status);
 		/* If any firmware (including CAP) load fails under SRIOV, it should
 		 * return failure to stop the VF from initializing.
+		 * A VCN firmware load failure must also stop initialization.
 		 * Also return failure in case of timeout
 		 */
-		if ((ucode && amdgpu_sriov_vf(psp->adev)) || !timeout) {
+		if ((ucode &&
+		    (amdgpu_sriov_vf(psp->adev) ||
+		    ucode->ucode_id == AMDGPU_UCODE_ID_VCN)) ||
+		    !timeout) {
 			ret = -EINVAL;
 			goto exit;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c
index 8ae72c862d11..2d494ac57eb4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c
@@ -23,6 +23,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_psp_ta.h"
+#include "amdgpu_ras_mgr.h"
 
 #if defined(CONFIG_DEBUG_FS)
 
@@ -330,6 +331,22 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size
 	if (IS_ERR(shared_buf))
 		return PTR_ERR(shared_buf);
 
+	if (ta_type == TA_TYPE_RAS && amdgpu_uniras_enabled(adev)) {
+		struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+
+		if (!ras_mgr || !ras_mgr->ras_core) {
+			ret = -EINVAL;
+			goto free_shared_buf;
+		}
+
+		ret = ras_psp_invoke_ras_ta(ras_mgr->ras_core, ta_id, cmd_id,
+					   shared_buf, shared_buf_len);
+		if (!ret && copy_to_user((char *)&buf[copy_pos], shared_buf, shared_buf_len))
+			ret = -EFAULT;
+
+		goto free_shared_buf;
+	}
+
 	set_ta_context_funcs(psp, ta_type, &context);
 
 	if (!context || !context->initialized) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 47e852c15c36..557baca454a9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -3616,6 +3616,9 @@ int amdgpu_ras_init(struct amdgpu_device *adev)
 
 	amdgpu_ras_check_supported(adev);
 
+	/* The RAS core derives the bad page threshold from this value. */
+	amdgpu_ras_init_reserved_vram_size(adev);
+
 	param.ras_feature_enable = amdgpu_ras_enable;
 	param.ras_feature_mask = amdgpu_ras_mask;
 	param.ras_bad_page_threshold = amdgpu_bad_page_threshold;
@@ -3713,8 +3716,6 @@ int amdgpu_ras_init(struct amdgpu_device *adev)
 	/* Get RAS schema for particular SOC */
 	con->schema = amdgpu_get_ras_schema(adev);
 
-	amdgpu_ras_init_reserved_vram_size(adev);
-
 	if (amdgpu_ras_fs_init(adev)) {
 		r = -EINVAL;
 		goto release_con;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
index 2ae2c1d0f702..e38a8ea02c52 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
@@ -292,7 +292,7 @@ struct amdgpu_rlc_funcs {
 	void (*reset)(struct amdgpu_device *adev);
 	void (*start)(struct amdgpu_device *adev);
 	void (*update_spm_vmid)(struct amdgpu_device *adev, int xcc_id,
-			struct amdgpu_ring *ring, unsigned vmid);
+			unsigned int vmid);
 	bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t reg);
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
index 21a225b0116a..fbf9ec7d467b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
@@ -103,13 +103,6 @@ int amdgpu_seq64_map(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		goto error;
 	}
 
-	r = amdgpu_vm_bo_update(adev, *bo_va, false);
-	if (r) {
-		DRM_ERROR("failed to do vm_bo_update on userq sem\n");
-		amdgpu_vm_bo_del(adev, *bo_va);
-		goto error;
-	}
-
 error:
 	drm_exec_fini(&exec);
 	return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 66f4a1872913..1373a11f50d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -566,22 +566,6 @@ TRACE_EVENT(amdgpu_ib_pipe_sync,
 		      __get_str(ring), __entry->ctx, __entry->seqno)
 );
 
-TRACE_EVENT(amdgpu_reset_reg_dumps,
-	    TP_PROTO(uint32_t address, uint32_t value),
-	    TP_ARGS(address, value),
-	    TP_STRUCT__entry(
-			     __field(uint32_t, address)
-			     __field(uint32_t, value)
-			     ),
-	    TP_fast_assign(
-			   __entry->address = address;
-			   __entry->value = value;
-			   ),
-	    TP_printk("amdgpu register dump 0x%x: 0x%x",
-		      __entry->address,
-		      __entry->value)
-);
-
 DECLARE_EVENT_CLASS(amdgpu_userq_queue,
 	    TP_PROTO(struct amdgpu_usermode_queue *queue),
 	    TP_ARGS(queue),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 8f0720a77520..0f86777250fc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -784,7 +784,8 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
 		goto out_unlock;
 	}
 	if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
-		vma->vm_file)) {
+		     (vma->vm_file ||
+		      vma->vm_end < start + (u64)ttm->num_pages * PAGE_SIZE))) {
 		r = -EPERM;
 		goto out_unlock;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
index 17888cf54447..6cd3927a7e82 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
@@ -137,14 +137,21 @@ static bool __check_vpod_info(struct amdgpu_device *adev,
 			 vpod->size, ppod->size);
 		return false;
 	}
-	if (vpod->addr_mode >= AMDGPU_UALINK_ADDR_MODE_MAX) {
+
+	switch (vpod->addr_mode) {
+	case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS:
+		break;
+	case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT:
 		dev_warn(adev->dev,
-			 "UALINK: invalid addr mode %u\n",
+			 "UALINK: source-identification addr mode is not supported\n");
+		return false;
+	default:
+		dev_warn(adev->dev, "UALINK: invalid addr mode %u\n",
 			 vpod->addr_mode);
 		return false;
 	}
-	weight =
-		bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
+
+	weight = bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
 	if (weight != vpod->size) {
 		dev_warn(adev->dev,
 			 "UALINK: vPod size doesn't match vpod_active_accels list: %u != %u\n",
@@ -1351,8 +1358,7 @@ static ssize_t ualink_vpod_config_commit_store(struct kobject *kobj,
 	if (info->vpod.id != AMDGPU_UALINK_VPOD_ID_INVALID &&
 	    !__check_vpod_info(adev, info)) {
 		dev_err(adev->dev,
-			"UALINK: vpod-commit __check_vpod_info() failed: %d\n",
-			r);
+			"UALINK: vpod-commit __check_vpod_info() failed\n");
 		return -EINVAL;
 	}
 
@@ -6075,9 +6081,8 @@ static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev,
 	struct amdgpu_ualink_ring *ring;
 	int r;
 
-	dev_dbg(adev->dev, "skip remote shootdown to remote_accel_id %u\n",
+	dev_dbg(adev->dev, "send remote shootdown to remote_accel_id %u\n",
 		remote_accel_id);
-	return 0;
 
 	peer = &remote->peer[remote_accel_id];
 	ring = &peer->shootdown;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 4dca1ab48410..1a2aa85c504e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -1077,6 +1077,13 @@ retry_lock:
 		if (unlikely(ret))
 			goto unlock_all;
 
+		/*
+		 * Userqueues never go through amdgpu_vm_flush(), so this is
+		 * the only place that can rebind the SPM VMID's page table
+		 * for a userqueue-held reservation.
+		 */
+		amdgpu_vm_spm_update(adev, vm);
+
 		/* This locks and validates the remaining evicted BOs */
 		ret = amdgpu_userq_bo_validate(adev, &exec, vm);
 		drm_exec_retry_on_contention(&exec);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index c1257f530e9a..e93e0e1c3144 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -384,7 +384,9 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
 			goto lock_error;
 
 		mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr);
-		if (!mapping || !mapping->bo_va->base.bo) {
+		/* Imported dma-bufs can't be CPU mapped with amdgpu_bo_kmap(). */
+		if (!mapping || !mapping->bo_va->base.bo ||
+		    drm_gem_is_imported(&mapping->bo_va->base.bo->tbo.base)) {
 			ret = -EINVAL;
 			goto lock_error;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 2bd07007def4..613c0b43ea3d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -230,7 +230,7 @@ static void amdgpu_vm_bo_needs_update(struct amdgpu_vm_bo_base *vm_bo)
 	 * idle state.
 	 */
 	lists = amdgpu_vm_bo_lock_lists(vm_bo);
-	if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
+	if (bo == vm_bo->vm->root.bo) {
 		vm_bo->moved = false;
 		list_move(&vm_bo->vm_status, &lists->idle);
 	} else {
@@ -569,6 +569,41 @@ uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 }
 
 /**
+ * amdgpu_vm_spm_update - bind the SPM VMID's page table base to @vm
+ *
+ * @adev: amdgpu device pointer
+ * @vm: vm to check against the SPM reservation
+ *
+ * The SPM VMID is fixed (adev->vm_manager.spm_vmid), so its page-table-base
+ * register must track whichever VM currently holds the reservation instead
+ * of being set per-job like a normal VMID. Bind it to @vm's real root PD
+ * address if @vm holds the reservation; otherwise do nothing, since @vm is
+ * not the one that could legitimately change the current binding.
+ *
+ * @vm's root PD must be locked by the caller.
+ */
+void amdgpu_vm_spm_update(struct amdgpu_device *adev, struct amdgpu_vm *vm)
+{
+	struct amdgpu_kq_vmid_mgr *id_mgr =
+		&adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)];
+	uint64_t pd_addr;
+
+	if (!adev->vm_manager.spm_vmid || !adev->gfxhub.funcs)
+		return;
+
+	mutex_lock(&id_mgr->lock);
+	if (vm->reserved_vmid[AMDGPU_GFXHUB(0)] !=
+	    &id_mgr->ids[adev->vm_manager.spm_vmid] || !amdgpu_vm_ready(vm)) {
+		mutex_unlock(&id_mgr->lock);
+		return;
+	}
+
+	pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);
+	amdgpu_gmc_bind_vmid(adev, AMDGPU_GFXHUB(0), adev->vm_manager.spm_vmid, pd_addr);
+	mutex_unlock(&id_mgr->lock);
+}
+
+/**
  * amdgpu_vm_validate - validate evicted BOs tracked in the VM
  *
  * @adev: amdgpu device pointer
@@ -766,14 +801,12 @@ bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
  * @ring: ring to use for flush
  * @job:  related job
  * @need_pipe_sync: is pipe sync needed
- * @emit_spm_needed: does the caller need to emit spm
  * @emit_gds_needed: does the caller need to emit gds
  *
  * Emit a VM flush when it is necessary.
  */
 void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
-		     bool *need_pipe_sync, bool *emit_spm_needed,
-		     bool *emit_gds_needed)
+		     bool *need_pipe_sync, bool *emit_gds_needed)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_isolation *isolation =
@@ -782,8 +815,6 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	unsigned vmhub = ring->vm_hub;
 	struct amdgpu_kq_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
-	bool spm_update_needed = adev->gfx.rlc.funcs->update_spm_vmid &&
-		job->spm_update_needed;
 	bool gds_switch_needed = ring->funcs->emit_gds_switch &&
 		job->gds_switch_needed;
 	bool vm_flush_needed = job->vm_needs_flush;
@@ -797,7 +828,6 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 		gds_switch_needed = true;
 		vm_flush_needed = true;
 		pasid_mapping_needed = true;
-		spm_update_needed = true;
 	}
 
 	mutex_lock(&id_mgr->lock);
@@ -807,7 +837,6 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	mutex_unlock(&id_mgr->lock);
 
 	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
-	spm_update_needed &= !!adev->gfx.rlc.funcs->update_spm_vmid;
 	vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&
 			job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
 	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
@@ -821,10 +850,6 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	emit_fence = vm_flush_needed || pasid_mapping_needed ||
 		cleaner_shader_needed;
 
-	*emit_spm_needed = spm_update_needed;
-	if (spm_update_needed && emit_fence)
-		*emit_spm_needed = false;
-
 	*emit_gds_needed = gds_switch_needed;
 	if (gds_switch_needed && emit_fence)
 		*emit_gds_needed = false;
@@ -864,9 +889,6 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 	if (pasid_mapping_needed)
 		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
 
-	if (spm_update_needed)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, ring->xcc_id, ring, job->vmid);
-
 	if (gds_switch_needed)
 		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
 						    job->gds_size, job->gws_base,
@@ -1802,7 +1824,6 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
 				    struct amdgpu_bo_va_mapping *mapping)
 {
 	struct amdgpu_vm *vm = bo_va->base.vm;
-	struct amdgpu_bo *bo = bo_va->base.bo;
 
 	mapping->bo_va = bo_va;
 	list_add(&mapping->list, &bo_va->invalids);
@@ -1811,7 +1832,8 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
 	if (mapping->flags & AMDGPU_VM_PAGE_PRT)
 		amdgpu_vm_prt_get(adev);
 
-	if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved)
+	/* PRT mappings have no BO and are kept off the state lists */
+	if (bo_va->base.bo && !bo_va->base.moved)
 		amdgpu_vm_bo_needs_update(&bo_va->base);
 
 	trace_amdgpu_vm_bo_map(bo_va, mapping);
@@ -2737,10 +2759,7 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 
 	if (vm->use_cpu_for_update) {
 		/* Sync with last SDMA update/clear before switching to CPU */
-		r = amdgpu_bo_sync_wait(vm->root.bo,
-					AMDGPU_FENCE_OWNER_UNDEFINED, true);
-		if (r)
-			goto unreserve_bo;
+		dma_fence_wait(vm->last_update, false);
 
 		vm->update_funcs = &amdgpu_vm_cpu_funcs;
 		r = amdgpu_vm_pt_map_tables(adev, vm);
@@ -2797,6 +2816,7 @@ int amdgpu_vm_make_npa(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 		return r;
 
 	vm->is_npa = true;
+	vm->need_tlb_fence = false;
 
 	return 0;
 }
@@ -2872,7 +2892,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	dma_fence_put(vm->last_update);
 
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) {
-		amdgpu_vmid_free_reserved(adev, vm, i);
+		amdgpu_vmid_free_spm(adev, vm, i);
 	}
 
 	ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);
@@ -2971,11 +2991,19 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		return -EINVAL;
 
 	switch (args->in.op) {
-	case AMDGPU_VM_OP_RESERVE_VMID:
+	case AMDGPU_VM_OP_RESERVE_VMID: {
+		int r;
+
 		/* We only have requirement to reserve vmid from gfxhub */
-		return amdgpu_vmid_alloc_reserved(adev, vm, AMDGPU_GFXHUB(0));
+		r = amdgpu_vmid_alloc_spm(adev, vm, AMDGPU_GFXHUB(0));
+		if (!r && !amdgpu_bo_reserve(vm->root.bo, true)) {
+			amdgpu_vm_spm_update(adev, vm);
+			amdgpu_bo_unreserve(vm->root.bo);
+		}
+		return r;
+	}
 	case AMDGPU_VM_OP_UNRESERVE_VMID:
-		amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(0));
+		amdgpu_vmid_free_spm(adev, vm, AMDGPU_GFXHUB(0));
 		break;
 	default:
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 8e478d3bda1a..2a2a2ccbaa2c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -383,7 +383,6 @@ struct amdgpu_vm {
 struct amdgpu_vm_manager {
 	/* Handling of VMIDs */
 	struct amdgpu_kq_vmid_mgr		id_mgr[AMDGPU_MAX_VMHUBS];
-	unsigned int				first_kfd_vmid;
 	bool					concurrent_flush;
 
 	uint64_t				max_pfn;
@@ -412,7 +411,9 @@ struct amdgpu_vm_manager {
 
 	/* Global registration of recent page fault information */
 	struct amdgpu_vm_fault_info		fault_info;
+	/* VMIDs dedicated to a single fixed purpose, 0 if unused */
 	unsigned int				npa_vmid;
+	unsigned int				spm_vmid;
 
 	/* VMID masks for GFXHUB and MMHUB */
 	uint32_t				vmid_uq_mask_gfxhub;
@@ -442,13 +443,13 @@ int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
 			      unsigned int num_fences);
 bool amdgpu_vm_ready(struct amdgpu_vm *vm);
 uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+void amdgpu_vm_spm_update(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		       struct ww_acquire_ctx *ticket,
 		       int (*callback)(void *p, struct amdgpu_bo *bo),
 		       void *param);
 void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
-		     bool *need_pipe_sync, bool *emit_spm_needed,
-		     bool *emit_gds_needed);
+		     bool *need_pipe_sync, bool *emit_gds_needed);
 int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 			  struct amdgpu_vm *vm, bool immediate);
 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index c03327f1242d..c73eb416c0e2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -427,7 +427,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (r)
 		goto exit;
 
-	r = vm->update_funcs->commit(&params, NULL);
+	r = vm->update_funcs->commit(&params, &vm->last_update);
 exit:
 	drm_dev_exit(idx);
 	return r;
@@ -531,13 +531,16 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p,
 	r2 = amdgpu_vm_begin_critical(p);
 	if (r)
 		return r;
-	if (r2)
-		goto error_free_pt;
+
+	pt_bo = &pt->bo;
+	if (r2) {
+		r = r2;
+		goto error_unpin;
+	}
 
 	/* Keep a reference to the root directory to avoid
 	 * freeing them up in the wrong order.
 	 */
-	pt_bo = &pt->bo;
 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
 	amdgpu_vm_bo_base_init(entry, p->vm, pt_bo);
 	r = amdgpu_vm_pt_clear(p->adev, p->vm, pt, p->immediate);
@@ -549,7 +552,6 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p,
 error_unpin:
 	if (p->vm->is_npa)
 		amdgpu_bo_unpin(pt_bo);
-error_free_pt:
 	amdgpu_bo_unref(&pt_bo);
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index ab14079d1c42..fda0ce61030d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -413,7 +413,6 @@ int amdgpu_xcp_dev_register(struct amdgpu_device *adev,
 
 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev)
 {
-	struct drm_device *p_ddev;
 	int i;
 
 	if (!adev->xcp_mgr)
@@ -423,12 +422,39 @@ void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev)
 		if (!adev->xcp_mgr->xcp[i].ddev)
 			break;
 
-		p_ddev = adev->xcp_mgr->xcp[i].ddev;
-		drm_dev_unplug(p_ddev);
-		p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev;
-		p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev;
-		p_ddev->driver =  adev->xcp_mgr->xcp[i].driver;
-		p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager;
+		drm_dev_unplug(adev->xcp_mgr->xcp[i].ddev);
+	}
+}
+
+void amdgpu_xcp_dev_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_xcp *xcp;
+	struct drm_device *p_ddev;
+	int i;
+
+	if (!adev->xcp_mgr)
+		return;
+
+	for (i = 1; i < MAX_XCP; i++) {
+		xcp = &adev->xcp_mgr->xcp[i];
+		if (!xcp->ddev)
+			break;
+
+		p_ddev = xcp->ddev;
+
+		/* Unregister devices on paths that did not unplug them earlier. */
+		if (p_ddev->registered)
+			drm_dev_unplug(p_ddev);
+
+		/*
+		 * All XCP minor files have been released, so it is now safe to
+		 * restore the partition device and free it.
+		 */
+		p_ddev->render->dev = xcp->rdev;
+		p_ddev->primary->dev = xcp->pdev;
+		p_ddev->driver = xcp->driver;
+		p_ddev->vma_offset_manager = xcp->vma_offset_manager;
+		xcp->ddev = NULL;
 		amdgpu_xcp_drm_dev_free(p_ddev);
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index 33157409eda0..e2fd32af9932 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -181,6 +181,7 @@ int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp,
 int amdgpu_xcp_dev_register(struct amdgpu_device *adev,
 				const struct pci_device_id *ent);
 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev);
+void amdgpu_xcp_dev_fini(struct amdgpu_device *adev);
 int amdgpu_xcp_open_device(struct amdgpu_device *adev,
 			   struct amdgpu_fpriv *fpriv,
 			   struct drm_file *file_priv);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 45e31b3daf06..ccd6429ad7b2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -349,6 +349,9 @@ int amdgpu_get_xgmi_link_status(struct amdgpu_device *adev, int global_link_num)
  * The XGMI memory space is built by contiguously adding the power of
  * two padded VRAM space from each node to each other.
  *
+ * Note that even if multiple GPUs have the same hive ID, sharing between their
+ * memory spaces via XGMI links can be disabled in firmware, i.e. hive
+ * membership does not guarantee connectivity.
  */
 
 static struct attribute amdgpu_xgmi_hive_id = {
@@ -1484,7 +1487,8 @@ bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
 {
 	return (amdgpu_use_xgmi_p2p && adev != bo_adev &&
 		adev->gmc.xgmi.hive_id &&
-		adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id);
+		adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id &&
+		amdgpu_xgmi_get_is_sharing_enabled(adev, bo_adev));
 }
 
 void amdgpu_xgmi_early_init(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index d29991a4af4c..e1029239b2c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1927,6 +1927,16 @@ static bool cik_need_reset_on_init(struct amdgpu_device *adev)
 {
 	u32 clock_cntl, pc;
 
+	/*
+	 * MacBookPros with switchable graphics put VRAM at 0 when the iGPU
+	 * is enabled.  Any later post (S3 resume, GPU reset) moves it back to
+	 * the vbios default, so reset and post on load to start with the
+	 * vbios layout.
+	 */
+	if (adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE &&
+	    !(RREG32(mmMC_VM_FB_LOCATION) & MC_VM_FB_LOCATION__FB_BASE_MASK))
+		return true;
+
 	if (adev->flags & AMD_IS_APU)
 		return false;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index 33403e16df5c..b6f61f4a08e5 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -5178,19 +5178,12 @@ static u32 gfx_v10_0_init_pa_sc_tile_steering_override(struct amdgpu_device *ade
 #define DEFAULT_SH_MEM_BASES	(0x6000)
 
 static void gfx_v10_0_debug_trap_config_init(struct amdgpu_device *adev,
-				uint32_t first_vmid,
-				uint32_t last_vmid)
+				uint32_t vmid_mask)
 {
 	uint32_t data;
-	uint32_t trap_config_vmid_mask = 0;
-	int i;
-
-	/* Calculate trap config vmid mask */
-	for (i = first_vmid; i < last_vmid; i++)
-		trap_config_vmid_mask |= (1 << i);
 
 	data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
-			VMID_SEL, trap_config_vmid_mask);
+			VMID_SEL, vmid_mask);
 	data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
 			TRAP_EN, 1);
 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
@@ -5214,7 +5207,7 @@ static void gfx_v10_0_init_compute_vmid(struct amdgpu_device *adev)
 	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		nv_grbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
@@ -5227,15 +5220,14 @@ static void gfx_v10_0_init_compute_vmid(struct amdgpu_device *adev)
 	 * Initialize all compute VMIDs to have no GDS, GWS, or OA
 	 * access. These should be enabled by FW for target VMIDs.
 	 */
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
 	}
 
-	gfx_v10_0_debug_trap_config_init(adev, adev->vm_manager.first_kfd_vmid,
-					AMDGPU_NUM_VMID);
+	gfx_v10_0_debug_trap_config_init(adev, adev->vm_manager.vmid_uq_mask_gfxhub);
 }
 
 static void gfx_v10_0_init_gds_vmid(struct amdgpu_device *adev)
@@ -5583,7 +5575,7 @@ static int gfx_v10_0_rlc_resume(struct amdgpu_device *adev)
 
 		gfx_v10_0_init_csb(adev);
 
-		gfx_v10_0_update_spm_vmid_internal(adev, 0xf);
+		gfx_v10_0_update_spm_vmid_internal(adev, adev->vm_manager.spm_vmid);
 
 		if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
 			gfx_v10_0_rlc_enable_srm(adev);
@@ -5615,7 +5607,7 @@ static int gfx_v10_0_rlc_resume(struct amdgpu_device *adev)
 
 		gfx_v10_0_init_csb(adev);
 
-		gfx_v10_0_update_spm_vmid_internal(adev, 0xf);
+		gfx_v10_0_update_spm_vmid_internal(adev, adev->vm_manager.spm_vmid);
 
 		adev->gfx.rlc.funcs->start(adev);
 
@@ -8328,7 +8320,7 @@ static void gfx_v10_0_update_spm_vmid_internal(struct amdgpu_device *adev,
 }
 
 static void gfx_v10_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
-		struct amdgpu_ring *ring, unsigned int vmid)
+		unsigned int vmid)
 {
 	amdgpu_gfx_off_ctrl(adev, false);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index d5f428ac19bf..315dfd806d56 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -944,7 +944,7 @@ static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
 
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid);
 
 	return 0;
 }
@@ -2167,7 +2167,7 @@ static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
 			SCRATCH_APP_BASE;
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		soc21_grbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
@@ -2185,7 +2185,7 @@ static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
 	 * Initialize all compute VMIDs to have no GDS, GWS, or OA
 	 * access. These should be enabled by FW for target VMIDs.
 	 */
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
 		WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
 		WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
@@ -5748,7 +5748,7 @@ static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
 }
 
 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
-		struct amdgpu_ring *ring, unsigned vmid)
+		unsigned int vmid)
 {
 	u32 reg, pre_data, data;
 
@@ -5769,14 +5769,6 @@ static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
 			WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
 	}
 	amdgpu_gfx_off_ctrl(adev, true);
-
-	if (ring
-		&& amdgpu_sriov_is_pp_one_vf(adev)
-		&& (pre_data != data)
-		&& ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
-			|| (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
-		amdgpu_ring_emit_wreg(ring, reg, data);
-	}
 }
 
 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index 420ce997bebc..d334fb805020 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -773,7 +773,7 @@ static int gfx_v12_0_rlc_init(struct amdgpu_device *adev)
 
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid);
 
 	return 0;
 }
@@ -1808,7 +1808,7 @@ static void gfx_v12_0_init_compute_vmid(struct amdgpu_device *adev)
 			SCRATCH_APP_BASE;
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		soc24_grbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
@@ -2714,109 +2714,6 @@ static int gfx_v12_0_cp_gfx_start(struct amdgpu_device *adev)
 		     adev->gfx.config.max_hw_contexts - 1);
 	WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
 
-	if (!amdgpu_async_gfx_ring)
-		gfx_v12_0_cp_gfx_enable(adev, true);
-
-	return 0;
-}
-
-static void gfx_v12_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
-					 CP_PIPE_ID pipe)
-{
-	u32 tmp;
-
-	tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
-	tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
-
-	WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
-}
-
-static void gfx_v12_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
-					  struct amdgpu_ring *ring)
-{
-	u32 tmp;
-
-	tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
-	if (ring->use_doorbell) {
-		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
-				    DOORBELL_OFFSET, ring->doorbell_index);
-		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
-				    DOORBELL_EN, 1);
-	} else {
-		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
-				    DOORBELL_EN, 0);
-	}
-	WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
-
-	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
-			    DOORBELL_RANGE_LOWER, ring->doorbell_index);
-	WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
-
-	WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
-		     CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
-}
-
-static int gfx_v12_0_cp_gfx_resume(struct amdgpu_device *adev)
-{
-	struct amdgpu_ring *ring;
-	u32 tmp;
-	u32 rb_bufsz;
-	u64 rb_addr, rptr_addr, wptr_gpu_addr;
-
-	/* Set the write pointer delay */
-	WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
-
-	/* set the RB to use vmid 0 */
-	WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
-
-	/* Init gfx ring 0 for pipe 0 */
-	mutex_lock(&adev->srbm_mutex);
-	gfx_v12_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
-
-	/* Set ring buffer size */
-	ring = &adev->gfx.gfx_ring[0];
-	rb_bufsz = order_base_2(ring->ring_size / 8);
-	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
-	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
-	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, KMD_QUEUE, 1);
-	WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
-
-	/* Initialize the ring buffer's write pointers */
-	ring->wptr = 0;
-	WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
-	WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
-
-	/* set the wb address whether it's enabled or not */
-	rptr_addr = ring->rptr_gpu_addr;
-	WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
-	WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
-		     CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
-
-	wptr_gpu_addr = ring->wptr_gpu_addr;
-	WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
-		     lower_32_bits(wptr_gpu_addr));
-	WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
-		     upper_32_bits(wptr_gpu_addr));
-
-	mdelay(1);
-	WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
-
-	rb_addr = ring->gpu_addr >> 8;
-	WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
-	WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
-
-	WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
-
-	gfx_v12_0_cp_gfx_set_doorbell(adev, ring);
-	mutex_unlock(&adev->srbm_mutex);
-
-	/* Switch to pipe 0 */
-	mutex_lock(&adev->srbm_mutex);
-	gfx_v12_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
-	mutex_unlock(&adev->srbm_mutex);
-
-	/* start the ring */
-	gfx_v12_0_cp_gfx_start(adev);
 	return 0;
 }
 
@@ -3520,9 +3417,6 @@ static int gfx_v12_0_kcq_resume(struct amdgpu_device *adev)
 {
 	int i, r;
 
-	if (!amdgpu_async_gfx_ring)
-		gfx_v12_0_cp_compute_enable(adev, true);
-
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 		r = gfx_v12_0_kcq_init_queue(&adev->gfx.compute_ring[i]);
 		if (r)
@@ -3553,10 +3447,8 @@ static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
 
 	gfx_v12_0_cp_set_doorbell_range(adev);
 
-	if (amdgpu_async_gfx_ring) {
-		gfx_v12_0_cp_compute_enable(adev, true);
-		gfx_v12_0_cp_gfx_enable(adev, true);
-	}
+	gfx_v12_0_cp_compute_enable(adev, true);
+	gfx_v12_0_cp_gfx_enable(adev, true);
 
 	if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) {
 		r = amdgpu_mes_kiq_hw_init(adev, 0);
@@ -3578,15 +3470,9 @@ static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
 	if (r)
 		return r;
 
-	if (!amdgpu_async_gfx_ring) {
-		r = gfx_v12_0_cp_gfx_resume(adev);
-		if (r)
-			return r;
-	} else {
-		r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
-		if (r)
-			return r;
-	}
+	r = gfx_v12_0_cp_async_gfx_ring_resume(adev);
+	if (r)
+		return r;
 
 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 		ring = &adev->gfx.gfx_ring[i];
@@ -3902,10 +3788,8 @@ static int gfx_v12_0_hw_fini(struct amdgpu_ip_block *ip_block)
 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 
 	if (!adev->no_hw_access) {
-		if (amdgpu_async_gfx_ring) {
-			if (amdgpu_gfx_disable_kgq(adev, 0))
-				DRM_ERROR("KGQ disable failed\n");
-		}
+		if (amdgpu_gfx_disable_kgq(adev, 0))
+			DRM_ERROR("KGQ disable failed\n");
 
 		if (amdgpu_gfx_disable_kcq(adev, 0))
 			DRM_ERROR("KCQ disable failed\n");
@@ -4075,7 +3959,6 @@ static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev,
 
 static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev,
 				      int xcc_id,
-				      struct amdgpu_ring *ring,
 				      unsigned vmid)
 {
 	u32 reg, data;
@@ -4093,14 +3976,6 @@ static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev,
 		WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
 	else
 		WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
-
-	if (ring
-	    && amdgpu_sriov_is_pp_one_vf(adev)
-	    && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
-		|| (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
-		uint32_t reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
-		amdgpu_ring_emit_wreg(ring, reg, data);
-	}
 }
 
 static const struct amdgpu_rlc_funcs gfx_v12_0_rlc_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
index 4498e253846f..1589fc6413af 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -37,6 +37,7 @@
 
 #include "gc/gc_12_1_0_offset.h"
 #include "gc/gc_12_1_0_sh_mask.h"
+#include "smuio/smuio_15_0_0_offset.h"
 #include "soc24_enum.h"
 #include "ivsrcid/gfx/irqsrcs_gfx_12_1_0.h"
 
@@ -214,7 +215,7 @@ static int gfx_v12_1_init_cp_compute_microcode_bo(struct amdgpu_device *adev);
 static void gfx_v12_1_xcc_update_medium_grain_clock_gating(
 	struct amdgpu_device *adev, bool enable, int xcc_id, bool force);
 static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
-				      struct amdgpu_ring *ring, unsigned vmid);
+				      unsigned int vmid);
 
 static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 				   int mem_space, int opt, uint32_t addr0,
@@ -1540,7 +1541,7 @@ static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev,
 	sh_mem_config = gfx_v12_1_get_sh_mem_config(adev);
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
 		/* CP and shaders */
 		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, sh_mem_config);
@@ -1861,7 +1862,7 @@ static int gfx_v12_1_xcc_rlc_resume(struct amdgpu_device *adev,
 	int r;
 
 	/* init spm vmid with 0xf */
-	gfx_v12_1_update_spm_vmid(adev, xcc_id, NULL, 0xf);
+	gfx_v12_1_update_spm_vmid(adev, xcc_id, adev->vm_manager.spm_vmid);
 
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 		gfx_v12_1_xcc_init_csb(adev, xcc_id);
@@ -2814,10 +2815,35 @@ static int gfx_v12_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return -ETIMEDOUT;
 }
 
+static uint64_t gfx_v12_1_get_smuio_gpu_clock_counter(struct amdgpu_device *adev)
+{
+	uint64_t clock = 0;
+	uint32_t reg;
+
+	if (!down_read_trylock(&adev->reset_domain->sem))
+		return 0;
+	if (amdgpu_device_skip_hw_access(adev) ||
+	    !adev->mes.ring[0].sched.ready)
+		goto unlock_reset;
+
+	reg = SOC15_REG_OFFSET(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
+	if (amdgpu_mes_rreg64(adev, reg, 0, &clock))
+		goto unlock_reset;
+
+	clock = (clock << 32) | (clock >> 32);
+
+unlock_reset:
+	up_read(&adev->reset_domain->sem);
+	return clock;
+}
+
 static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev)
 {
 	uint64_t clock = 0;
 
+	if (amdgpu_sriov_vf(adev))
+		return gfx_v12_1_get_smuio_gpu_clock_counter(adev);
+
 	if (adev->smuio.funcs &&
 	    adev->smuio.funcs->get_gpu_clock_counter)
 		clock = adev->smuio.funcs->get_gpu_clock_counter(adev);
@@ -2918,8 +2944,7 @@ static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
 
 static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev,
 				      int xcc_id,
-				      struct amdgpu_ring *ring,
-				      unsigned vmid)
+				      unsigned int vmid)
 {
 	u32 reg, data;
 
@@ -2936,14 +2961,6 @@ static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev,
 		WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data);
 	else
 		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data);
-
-	if (ring
-	    && amdgpu_sriov_is_pp_one_vf(adev)
-	    && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
-		|| (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
-		uint32_t reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL);
-		amdgpu_ring_emit_wreg(ring, reg, data);
-	}
 }
 
 static const struct amdgpu_rlc_funcs gfx_v12_1_rlc_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 549d1239e326..50e25109d0eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1837,7 +1837,7 @@ static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)
 			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
 	sh_mem_config |= MTYPE_NONCACHED << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT;
 	mutex_lock(&adev->srbm_mutex);
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		cik_srbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		WREG32(mmSH_MEM_CONFIG, sh_mem_config);
@@ -1850,7 +1850,7 @@ static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)
 
 	/* Initialize all compute VMIDs to have no GDS, GWS, or OA
 	   access. These should be enabled by FW for target VMIDs. */
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
 		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
 		WREG32(amdgpu_gds_reg_offset[i].gws, 0);
@@ -3252,7 +3252,7 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
 
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid);
 
 	return 0;
 }
@@ -3479,7 +3479,7 @@ static int gfx_v7_0_rlc_resume(struct amdgpu_device *adev)
 }
 
 static void gfx_v7_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
-		struct amdgpu_ring *ring, unsigned vmid)
+		unsigned int vmid)
 {
 	u32 data;
 
@@ -4460,14 +4460,6 @@ static int gfx_v7_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	amdgpu_gfx_mqd_sw_fini(adev, 0);
 	amdgpu_gfx_rlc_fini(adev);
 	gfx_v7_0_mec_fini(adev);
-	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
-				&adev->gfx.rlc.clear_state_gpu_addr,
-				(void **)&adev->gfx.rlc.cs_ptr);
-	if (adev->gfx.rlc.cp_table_size) {
-		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
-				&adev->gfx.rlc.cp_table_gpu_addr,
-				(void **)&adev->gfx.rlc.cp_table_ptr);
-	}
 	gfx_v7_0_free_microcode(adev);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index ab4123ddbb94..7e2f9c13dec3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -1274,7 +1274,7 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid);
 
 	return 0;
 }
@@ -2062,15 +2062,6 @@ static int gfx_v8_0_sw_fini(struct amdgpu_ip_block *ip_block)
 
 	gfx_v8_0_mec_fini(adev);
 	amdgpu_gfx_rlc_fini(adev);
-	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
-				&adev->gfx.rlc.clear_state_gpu_addr,
-				(void **)&adev->gfx.rlc.cs_ptr);
-	if ((adev->asic_type == CHIP_CARRIZO) ||
-	    (adev->asic_type == CHIP_STONEY)) {
-		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
-				&adev->gfx.rlc.cp_table_gpu_addr,
-				(void **)&adev->gfx.rlc.cp_table_ptr);
-	}
 	gfx_v8_0_free_microcode(adev);
 
 	return 0;
@@ -3682,7 +3673,7 @@ static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
 			SH_MEM_CONFIG__PRIVATE_ATC_MASK;
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		vi_srbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		WREG32(mmSH_MEM_CONFIG, sh_mem_config);
@@ -3695,7 +3686,7 @@ static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
 
 	/* Initialize all compute VMIDs to have no GDS, GWS, or OA
 	   access. These should be enabled by FW for target VMIDs. */
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
 		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
 		WREG32(amdgpu_gds_reg_offset[i].gws, 0);
@@ -5491,7 +5482,7 @@ static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
 }
 
 static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
-		struct amdgpu_ring *ring, unsigned vmid)
+		unsigned int vmid)
 {
 	u32 data;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 1678015ce5a9..75d1bbdb421f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -2568,19 +2568,12 @@ static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
 }
 
 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
-				uint32_t first_vmid,
-				uint32_t last_vmid)
+				uint32_t vmid_mask)
 {
 	uint32_t data;
-	uint32_t trap_config_vmid_mask = 0;
-	int i;
-
-	/* Calculate trap config vmid mask */
-	for (i = first_vmid; i < last_vmid; i++)
-		trap_config_vmid_mask |= (1 << i);
 
 	data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
-			VMID_SEL, trap_config_vmid_mask);
+			VMID_SEL, vmid_mask);
 	data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
 			TRAP_EN, 1);
 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
@@ -2610,7 +2603,7 @@ static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
 			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		soc15_grbm_select(adev, 0, 0, 0, i, 0);
 		/* CP and shaders */
 		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
@@ -2621,7 +2614,7 @@ static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
 
 	/* Initialize all compute VMIDs to have no GDS, GWS, or OA
 	   access. These should be enabled by FW for target VMIDs. */
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
@@ -3240,7 +3233,7 @@ static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
 		break;
 	}
 
-	gfx_v9_0_update_spm_vmid_internal(adev, 0xf);
+	gfx_v9_0_update_spm_vmid_internal(adev, adev->vm_manager.spm_vmid);
 
 	adev->gfx.rlc.funcs->start(adev);
 
@@ -4932,11 +4925,10 @@ static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block)
 		return r;
 
 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
-		gfx_v9_4_2_debug_trap_config_init(adev,
-			adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
+		gfx_v9_4_2_debug_trap_config_init(adev);
 	else
 		gfx_v9_0_debug_trap_config_init(adev,
-			adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
+			adev->vm_manager.vmid_uq_mask_gfxhub);
 
 	return 0;
 }
@@ -5233,7 +5225,7 @@ static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
 }
 
 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
-		struct amdgpu_ring *ring, unsigned int vmid)
+		unsigned int vmid)
 {
 	amdgpu_gfx_off_ctrl(adev, false);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
index 54bcb5fdca24..f54f54fe535e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
@@ -836,16 +836,14 @@ void gfx_v9_4_2_init_sq(struct amdgpu_device *adev)
 	}
 }
 
-void gfx_v9_4_2_debug_trap_config_init(struct amdgpu_device *adev,
-				uint32_t first_vmid,
-				uint32_t last_vmid)
+void gfx_v9_4_2_debug_trap_config_init(struct amdgpu_device *adev)
 {
 	uint32_t data;
 	int i;
 
 	mutex_lock(&adev->srbm_mutex);
 
-	for (i = first_vmid; i < last_vmid; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		data = 0;
 		soc15_grbm_select(adev, 0, 0, 0, i, 0);
 		data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
index c8757de669ea..ca240f4298da 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
@@ -24,8 +24,7 @@
 #ifndef __GFX_V9_4_2_H__
 #define __GFX_V9_4_2_H__
 
-void gfx_v9_4_2_debug_trap_config_init(struct amdgpu_device *adev,
-				uint32_t first_vmid, uint32_t last_vmid);
+void gfx_v9_4_2_debug_trap_config_init(struct amdgpu_device *adev);
 void gfx_v9_4_2_init_golden_registers(struct amdgpu_device *adev,
 				      uint32_t die_id);
 void gfx_v9_4_2_init_sq(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index 01ed1701c781..d66c7901783e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -171,6 +171,7 @@ static int gfx_v9_4_3_get_cu_info(struct amdgpu_device *adev,
 				struct amdgpu_cu_info *cu_info);
 static void gfx_v9_4_3_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
 static void gfx_v9_4_3_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
+static void gfx_v9_4_3_enable_ptl_skip_interrupt(struct amdgpu_device *adev);
 
 static void gfx_v9_4_3_kiq_set_resources(struct amdgpu_ring *kiq_ring,
 				uint64_t queue_mask)
@@ -1173,7 +1174,7 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev,
 			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
 		/* CP and shaders */
 		WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, sh_mem_config);
@@ -1191,7 +1192,7 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev,
 	 * Initialize all compute VMIDs to have no GDS, GWS, or OA
 	 * access. These should be enabled by FW for target VMIDs.
 	 */
-	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+	for_each_uq_gfx_vmid(i, adev) {
 		WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_BASE, 2 * i, 0);
 		WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_SIZE, 2 * i, 0);
 		WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_GWS_VMID0, i, 0);
@@ -1395,7 +1396,7 @@ static int gfx_v9_4_3_rlc_init(struct amdgpu_device *adev)
 {
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, adev->vm_manager.spm_vmid);
 
 	return 0;
 }
@@ -1607,7 +1608,7 @@ static int gfx_v9_4_3_rlc_resume(struct amdgpu_device *adev)
 }
 
 static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev,
-					      int inst, struct amdgpu_ring *ring, unsigned int vmid)
+					      int inst, unsigned int vmid)
 {
 	u32 reg, pre_data, data;
 
@@ -2465,6 +2466,11 @@ static int gfx_v9_4_3_ptl_hw_init(struct amdgpu_device *adev)
 
 	gfx_v9_4_3_perf_monitor_ptl_init(adev, enable ? 1 : 0);
 
+	/* Enable PTL register skip notifications if PTL is supported */
+	if (ptl->hw_supported_state == AMDGPU_PTL_HW_SUPPORTED)
+		if (adev->gfx.mec_fw_version >= 194)
+			gfx_v9_4_3_enable_ptl_skip_interrupt(adev);
+
 	return 0;
 }
 
@@ -3327,6 +3333,34 @@ static int gfx_v9_4_3_set_bad_op_fault_state(struct amdgpu_device *adev,
 	return 0;
 }
 
+/*
+ * Enable GENERIC1 interrupt (bit 30) for PTL register skip notifications.
+ * When FW skips a privileged register access due to PTL, it signals via
+ * GENERIC1 and KFD receives interrupt source 177 with skip details.
+ */
+static void gfx_v9_4_3_enable_ptl_skip_interrupt(struct amdgpu_device *adev)
+{
+	u32 mec_int_cntl_reg, mec_int_cntl;
+	int i, j, k, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		for (j = 0; j < adev->gfx.mec.num_mec; j++) {
+			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+				mec_int_cntl_reg = gfx_v9_4_3_get_cpc_int_cntl(adev, i, j + 1, k);
+
+				if (mec_int_cntl_reg) {
+					mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, i);
+					mec_int_cntl = REG_SET_FIELD(mec_int_cntl,
+							CP_ME1_PIPE0_INT_CNTL,
+							GENERIC1_INT_ENABLE, 1);
+					WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, i);
+				}
+			}
+		}
+	}
+}
+
 static int gfx_v9_4_3_set_priv_inst_fault_state(struct amdgpu_device *adev,
 					      struct amdgpu_irq_src *source,
 					      unsigned type,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index 490b4c4eb662..49f593fe2a87 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -866,14 +866,15 @@ static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
-	 * amdgpu graphics/compute will use VMIDs 1-7
-	 * amdkfd will use VMIDs 8-15
+	 * amdgpu graphics/compute kernel queues will use VMIDs 1-7
+	 * amdkfd/MES user queues will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = 8;
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      false);
-	amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true);
+
+	/* Reserve the last VMID for SPM */
+	adev->vm_manager.spm_vmid = AMDGPU_NUM_VMID - 1;
+
+	amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(7, 1),
+				      GENMASK(AMDGPU_NUM_VMID - 1, 1));
 
 	amdgpu_vm_manager_init(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
index 3cdc7893587a..709c3d1b9998 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
@@ -745,6 +745,7 @@ static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
 	struct amdgpu_device *adev = ip_block->adev;
+	unsigned int last_kq_vmid;
 
 	adev->mmhub.funcs->init(adev);
 
@@ -851,15 +852,16 @@ static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
-	 * amdgpu graphics/compute will use VMIDs 1-7
-	 * amdkfd will use VMIDs 8-15
+	 * amdgpu graphics/compute kernel queues will use VMIDs 1-7
+	 * amdkfd/MES user queues will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
+	last_kq_vmid = adev->gfx.disable_kq ? 0 : 7;
 
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      false);
-	amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true);
+	/* Reserve the last VMID for SPM */
+	adev->vm_manager.spm_vmid = AMDGPU_NUM_VMID - 1;
+
+	amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(last_kq_vmid, 1),
+				      GENMASK(AMDGPU_NUM_VMID - 1, 1));
 
 	amdgpu_vm_manager_init(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index fcc1d580fa94..cd276a138e10 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -848,7 +848,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
 	struct amdgpu_device *adev = ip_block->adev;
 	uint64_t pte_addr_mask = 0;
-	u32 mmhub_vmid_mask;
+	unsigned int last_kq_vmid;
 	int i;
 
 	adev->mmhub.funcs->init(adev);
@@ -990,32 +990,29 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
-	 * amdgpu graphics/compute will use VMIDs 1-7
-	 * amdkfd will use VMIDs 8-15.
-	 * On GFX 12.1, amdkfd will use VMIDs 3-15, with VMID 15 reserved
-	 * for NPA and therefore excluded from the MMHUB VMID pool.
+	 * amdgpu graphics/compute kernel queues will use VMIDs 1-7
+	 * amdkfd/MES user queues will use VMIDs 8-15.
+	 * On GFX 12.1, amdkfd/MES user queues will use VMIDs 3-15, with
+	 * VMID 15 reserved for NPA and therefore excluded from the
+	 * MMHUB VMID pool.
 	 */
-	mmhub_vmid_mask = GENMASK(AMDGPU_NUM_VMID - 1, 1);
 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
-		adev->vm_manager.first_kfd_vmid = 3;
+		last_kq_vmid = 2;
 		adev->vm_manager.npa_vmid = 15;
-		mmhub_vmid_mask &= ~BIT(adev->vm_manager.npa_vmid);
 	} else {
-		adev->vm_manager.first_kfd_vmid = 8;
+		last_kq_vmid = 7;
 	}
 
 	if (adev->gfx.disable_kq)
-		adev->vm_manager.first_kfd_vmid = 1;
+		last_kq_vmid = 0;
 
-	amdgpu_vmid_mgr_set_vmid_mask(adev, mmhub_vmid_mask, true);
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      false);
+	/* Reserve the last VMID for SPM */
+	adev->vm_manager.spm_vmid = AMDGPU_NUM_VMID - 1;
 
-	amdgpu_vm_manager_init(adev);
+	amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(last_kq_vmid, 1),
+				      GENMASK(AMDGPU_NUM_VMID - 1, 1));
 
-	if (adev->vm_manager.npa_vmid)
-		adev->vm_manager.vmid_uq_mask_mmhub &= ~BIT(adev->vm_manager.npa_vmid);
+	amdgpu_vm_manager_init(adev);
 
 	r = amdgpu_gmc_ras_sw_init(adev);
 	if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 71409eb718d3..a7c353d1ac09 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -768,6 +768,7 @@ static int gmc_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	int r;
 	struct amdgpu_device *adev = ip_block->adev;
+	unsigned long vmid_mask;
 
 	set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
 
@@ -821,16 +822,11 @@ static int gmc_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
-	 * amdgpu graphics/compute will use VMIDs 1-7
-	 * amdkfd will use VMIDs 8-15
+	 * amdgpu graphics/compute kernel queues will use VMIDs 1-7
+	 * amdkfd/MES user queues will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = 8;
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      false);
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      true);
+	vmid_mask = GENMASK(7, 1);
+	amdgpu_vmid_mgr_set_vmid_mask(adev, vmid_mask, vmid_mask);
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index c95e097241bf..c09f5153679e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -941,6 +941,7 @@ static int gmc_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	int r;
 	struct amdgpu_device *adev = ip_block->adev;
+	unsigned long vmid_mask;
 
 	set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
 
@@ -1003,16 +1004,15 @@ static int gmc_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
-	 * amdgpu graphics/compute will use VMIDs 1-7
-	 * amdkfd will use VMIDs 8-15
+	 * amdgpu graphics/compute kernel queues will use VMIDs 1-7
+	 * amdkfd/MES user queues will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = 8;
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      false);
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      true);
+	vmid_mask = GENMASK(7, 1);
+
+	/* Reserve the last VMID for SPM */
+	adev->vm_manager.spm_vmid = AMDGPU_NUM_VMID - 1;
+
+	amdgpu_vmid_mgr_set_vmid_mask(adev, vmid_mask, vmid_mask);
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 8d8a0775878f..15592027e7c0 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -1051,6 +1051,7 @@ static int gmc_v8_0_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	int r;
 	struct amdgpu_device *adev = ip_block->adev;
+	unsigned long vmid_mask;
 
 	set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
 
@@ -1118,16 +1119,15 @@ static int gmc_v8_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
-	 * amdgpu graphics/compute will use VMIDs 1-7
-	 * amdkfd will use VMIDs 8-15
+	 * amdgpu graphics/compute kernel queues will use VMIDs 1-7
+	 * amdkfd/MES user queues will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = 8;
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      false);
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      true);
+	vmid_mask = GENMASK(7, 1);
+
+	/* Reserve the last VMID for SPM */
+	adev->vm_manager.spm_vmid = AMDGPU_NUM_VMID - 1;
+
+	amdgpu_vmid_mgr_set_vmid_mask(adev, vmid_mask, vmid_mask);
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 85ba228f002d..03395f59a6a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1849,6 +1849,8 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
 	struct amdgpu_device *adev = ip_block->adev;
 	unsigned long inst_mask = adev->aid_mask;
+	unsigned int last_kq_vmid;
+	unsigned long vmid_mask;
 
 	adev->gfxhub.funcs->init(adev);
 
@@ -2017,8 +2019,8 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
-	 * amdgpu graphics/compute will use VMIDs 1..n-1
-	 * amdkfd will use VMIDs n..15
+	 * amdgpu graphics/compute kernel queues will use VMIDs 1..n-1
+	 * amdkfd/MES user queues will use VMIDs n..15
 	 *
 	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
 	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
@@ -2030,21 +2032,21 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
 	    adev->jpeg.disable_kq &&
 	    adev->vcn.disable_kq &&
 	    adev->sdma.no_user_submission)
-		adev->vm_manager.first_kfd_vmid = 1;
+		last_kq_vmid = 0;
 	else
-		adev->vm_manager.first_kfd_vmid =
+		last_kq_vmid =
 			(amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
 			 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
 			 amdgpu_is_multi_aid(adev)) ?
-			3 :
-			8;
-
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      false);
-	amdgpu_vmid_mgr_set_vmid_mask(adev,
-				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
-				      true);
+			2 :
+			7;
+
+	vmid_mask = GENMASK(last_kq_vmid, 1);
+
+	/* Reserve the last VMID for SPM */
+	adev->vm_manager.spm_vmid = AMDGPU_NUM_VMID - 1;
+
+	amdgpu_vmid_mgr_set_vmid_mask(adev, vmid_mask, vmid_mask);
 	amdgpu_vm_manager_init(adev);
 
 	gmc_v9_0_save_registers(adev);
@@ -2179,10 +2181,22 @@ static int gmc_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
 	 * (type 2), which flushes both. Due to a race condition with
 	 * concurrent memory accesses using the same TLB cache line, we still
 	 * need a second TLB flush after this.
+	 *
+	 * GC 9.4.3, 9.4.4 and 9.5.0 hit a similar race.
 	 */
-	adev->gmc.flush_tlb_needs_extra_type_2 =
-		amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0) &&
-		adev->gmc.xgmi.num_physical_nodes;
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(9, 4, 0):
+		adev->gmc.flush_tlb_needs_extra_type_2 =
+			adev->gmc.xgmi.num_physical_nodes;
+		break;
+	case IP_VERSION(9, 4, 3):
+	case IP_VERSION(9, 4, 4):
+	case IP_VERSION(9, 5, 0):
+		adev->gmc.flush_tlb_needs_extra_type_2 = true;
+		break;
+	default:
+		break;
+	}
 
 	/* The sequence of these two function calls matters.*/
 	gmc_v9_0_init_golden_registers(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 1b76c10d1ac2..7abb9d95ef59 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -54,7 +54,12 @@ mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
 			goto fail_lock;
 
 		wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr);
-		if (!wptr_mapping || !wptr_mapping->bo_va->base.bo) {
+		/*
+		 * The WPTR is read through a kernel CPU mapping, which imported
+		 * dma-bufs don't support.
+		 */
+		if (!wptr_mapping || !wptr_mapping->bo_va->base.bo ||
+		    drm_gem_is_imported(&wptr_mapping->bo_va->base.bo->tbo.base)) {
 			ret = -EINVAL;
 			goto fail_lock;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index 78c517f6c2d2..b2e4da70d208 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -992,9 +992,15 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
 
 	switch (input->op) {
 	case MES_MISC_OP_READ_REG:
+		if (input->read_reg.read64_bits &&
+		    (mes->sched_version & AMDGPU_MES_VERSION_MASK) < 0x42)
+			return -EOPNOTSUPP;
+
 		misc_pkt.opcode = MESAPI_MISC__READ_REG;
 		misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
 		misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
+		misc_pkt.read_reg.option.bits.read64Bits =
+			input->read_reg.read64_bits;
 		break;
 	case MES_MISC_OP_WRITE_REG:
 		misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
@@ -1864,6 +1870,7 @@ static int mes_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int pipe, r, bo_size;
 
+	adev->mes.use_rs64mem = false;
 	adev->mes.funcs = &mes_v11_0_funcs;
 	adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;
 	adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;
@@ -1914,6 +1921,14 @@ static int mes_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 		return r;
 	}
 
+	/* Allocate GPU buffer for array size query results */
+	r = amdgpu_mes_rs64mem_init(&adev->mes);
+	if (r) {
+		dev_warn(adev->dev,
+			 "RS64 memory init failed (%d), using system memory\n", r);
+		adev->mes.use_rs64mem = false;
+	}
+
 	return 0;
 }
 
@@ -2091,8 +2106,6 @@ static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
 	if (adev->mes.ring[0].sched.ready)
 		goto out;
 
-	adev->mes.use_rs64mem = false;
-
 	if (!adev->enable_mes_kiq) {
 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 			r = mes_v11_0_load_microcode(adev,
@@ -2110,14 +2123,6 @@ static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		goto failure;
 
-	/* Allocate GPU buffer for array size query results */
-	r = amdgpu_mes_rs64mem_init(&adev->mes);
-	if (r) {
-		dev_warn(adev->dev,
-			 "RS64 local memory init failed (%d),"
-			 "falling back to system memory path\n", r);
-		adev->mes.use_rs64mem = false;
-	}
 	r = mes_v11_0_set_hw_resources(&adev->mes);
 
 	if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
index b6356095b079..ce577f49b630 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
@@ -947,6 +947,8 @@ static int mes_v12_0_misc_op(struct amdgpu_mes *mes,
 		misc_pkt.opcode = MESAPI_MISC__READ_REG;
 		misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
 		misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
+		misc_pkt.read_reg.option.bits.read64Bits =
+			input->read_reg.read64_bits;
 		break;
 	case MES_MISC_OP_WRITE_REG:
 		misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
@@ -991,6 +993,10 @@ static int mes_v12_0_misc_op(struct amdgpu_mes *mes,
 				input->change_config.option.limit_single_process;
 		break;
 	case MES_MISC_OP_SETUP_MES_DBGEXT:
+		/* The log buffer attaches to the pipe that receives this packet. */
+		if (input->setup_mes_dbgext.pipe >= AMDGPU_MAX_MES_PIPES)
+			return -EINVAL;
+		pipe = input->setup_mes_dbgext.pipe;
 		misc_pkt.opcode = MESAPI_MISC__SETUP_MES_DBGEXT;
 		misc_pkt.dbgext_init_data.dbg_ext_mc_addr =
 				input->setup_mes_dbgext.log_buffer_mc_addr;
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
index ef81c5a23b9d..99e8a7eeaa4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
@@ -572,6 +572,8 @@ static int mes_v12_1_misc_op(struct amdgpu_mes *mes,
 	case MES_MISC_OP_READ_REG:
 		misc_pkt.opcode = MESAPI_MISC__READ_REG;
 		misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
+		misc_pkt.read_reg.option.bits.read64Bits =
+			input->read_reg.read64_bits;
 		mes_v12_1_get_rrmt(input->read_reg.reg_offset,
 				   GET_INST(GC, input->xcc_id),
 				   &misc_pkt.read_reg.rrmt_opt,
diff --git a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
index a74b17a47fd2..dd37eae6ae94 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
@@ -30,32 +30,84 @@
 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
 #include <uapi/linux/kfd_ioctl.h>
 
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL_nbif_4_10                                 0x4f0aeb
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL_nbif_4_10_BASE_IDX                        3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1_nbif_4_10                                0x4f0aec
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1_nbif_4_10_BASE_IDX                       3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_nbif_4_10                                 0x4f0aed
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_nbif_4_10_BASE_IDX                        3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1_nbif_4_10                                0x4f0aee
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1_nbif_4_10_BASE_IDX                       3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_nbif_4_10                                 0x4f0aef
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_nbif_4_10_BASE_IDX                        3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1_nbif_4_10                                0x4f0af0
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1_nbif_4_10_BASE_IDX                       3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL_nbif_4_10                                 0x4f0af1
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL_nbif_4_10_BASE_IDX                        3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1_nbif_4_10                                0x4f0af2
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1_nbif_4_10_BASE_IDX                       3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10                                 0x4f0af3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10_BASE_IDX                        3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1_nbif_4_10                                0x4f0af4
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1_nbif_4_10_BASE_IDX                       3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_nbif_4_10                                 0x4f0af5
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_nbif_4_10_BASE_IDX                        3
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1_nbif_4_10                                0x4f0af6
-#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1_nbif_4_10_BASE_IDX                       3
-#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10                                    0x0021
-#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10_BASE_IDX                           2
+static const struct amdgpu_nbio_doorbell_layout
+nbif_v6_3_1_doorbell_layout = {
+	.ctrl = {
+		.enable = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_ENABLE),
+		.awid = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWID),
+		.fence_enable = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_FENCE_ENABLE),
+		.range_offset = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_OFFSET),
+		.range_size = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_SIZE),
+		.disable_64bit = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS),
+		.deduct_range_offset = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET),
+		.drop = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_DROP_EN),
+		.awaddr_31_28 = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE),
+	},
+};
+
+static void nbif_v6_3_1_init_doorbell_entry_registers(
+	struct amdgpu_device *adev)
+{
+	struct amdgpu_nbio_doorbell_entries *entries;
+	u32 ctrl_base;
+
+	memset(adev->nbio.doorbell_entries, 0,
+	       sizeof(adev->nbio.doorbell_entries));
+	adev->nbio.doorbell_layout = &nbif_v6_3_1_doorbell_layout;
+
+	ctrl_base = SOC15_REG_OFFSET(
+		NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL);
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP];
+	entries->num_entries = 1;
+	entries->entry[0].port = 0;
+	entries->entry[0].ctrl_reg = ctrl_base;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH];
+	entries->num_entries = 1;
+	entries->entry[0].port = 1;
+	entries->entry[0].ctrl_reg = ctrl_base + 1;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA];
+	entries->num_entries = 1;
+	entries->entry[0].port = 2;
+	entries->entry[0].ctrl_reg = ctrl_base + 2;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_RLC];
+	entries->num_entries = 1;
+	entries->entry[0].port = 3;
+	entries->entry[0].ctrl_reg = ctrl_base + 3;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN];
+	entries->num_entries = 2;
+	entries->entry[0].port = 4;
+	entries->entry[0].ctrl_reg = ctrl_base + 4;
+	entries->entry[1].port = 5;
+	entries->entry[1].ctrl_reg = ctrl_base + 5;
+}
 
 static void nbif_v6_3_1_remap_hdp_registers(struct amdgpu_device *adev)
 {
@@ -69,10 +121,7 @@ static u32 nbif_v6_3_1_get_rev_id(struct amdgpu_device *adev)
 {
 	u32 tmp;
 
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
-		tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10);
-	else
-		tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0);
+	tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0);
 
 	tmp &= RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK;
 	tmp >>= RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT;
@@ -100,153 +149,82 @@ static void nbif_v6_3_1_sdma_doorbell_range(struct amdgpu_device *adev,
 					    int doorbell_index,
 					    int doorbell_size)
 {
-	if (instance == 0) {
-		u32 doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL);
-
-		if (use_doorbell) {
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_ENABLE,
-						       0x1);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_AWID,
-						       0xe);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_OFFSET,
-						       doorbell_index);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_SIZE,
-						       doorbell_size);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE,
-						       0x3);
-		} else
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_SIZE,
-						       0);
-
-		if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
-			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_nbif_4_10,
-				     doorbell_range);
-		else
-			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-				     doorbell_range);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0xe,
+		.range_offset = doorbell_index,
+		.range_size = doorbell_size,
+		.awaddr_31_28 = 0x3,
+	};
+	u32 ctrl;
+
+	if (instance != 0)
+		return;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance,
+			NULL, NULL);
+		return;
 	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance, &ctrl, NULL);
 }
 
 static void nbif_v6_3_1_vcn_doorbell_range(struct amdgpu_device *adev,
 					   bool use_doorbell, int doorbell_index,
 					   int instance)
 {
-	u32 doorbell_range;
-
-	if (instance) {
-		if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
-			return;
-		doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL);
-	} else {
-		doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL);
-	}
-
-	if (use_doorbell) {
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_ENABLE,
-					       0x1);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_AWID,
-					       instance ? 0x7 : 0x4);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_RANGE_OFFSET,
-					       doorbell_index);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_RANGE_SIZE,
-					       8);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE,
-					       instance ? 0x7 : 0x4);
-	} else
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_RANGE_SIZE,
-					       0);
-
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) {
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10,
-			     doorbell_range);
-	} else {
-		if (instance)
-			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
-		else
-			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL, doorbell_range);
-	}
-}
-
-static void nbif_v6_3_1_vpe_doorbell_range(struct amdgpu_device *adev,
-							int instance, bool use_doorbell,
-							int doorbell_index,
-							int doorbell_size)
-{
-	if (instance)
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_offset = doorbell_index,
+		.range_size = 8,
+	};
+	u32 ctrl, entry = !!instance;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_VCN, entry,
+			NULL, NULL);
 		return;
-
-	u32 doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL);
-
-	if (use_doorbell) {
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_ENABLE,
-					       0x1);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_AWID,
-					       0xf);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_RANGE_OFFSET,
-					       doorbell_index);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_RANGE_SIZE,
-					       doorbell_size);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE,
-					       0xf);
-	} else {
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_RANGE_SIZE,
-					       0);
 	}
 
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4) ||
-		amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) {
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_nbif_4_10, doorbell_range);
-	} else {
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
-	}
+	fields.awid = instance ? 0x7 : 0x4;
+	fields.awaddr_31_28 = fields.awid;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
 
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN, entry, &ctrl, NULL);
 }
 
 static void nbif_v6_3_1_gc_doorbell_init(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) {
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL_nbif_4_10, 0x30000007);
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL_nbif_4_10, 0x3000000d);
-	} else {
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL, 0x30000007);
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL, 0x3000000d);
-	}
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awaddr_31_28 = 0x3,
+	};
+	u32 ctrl;
+
+	fields.awid = 0x3;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP, 0, &ctrl, NULL);
+
+	fields.awid = 0x6;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_RLC, 0, &ctrl, NULL);
 }
 
 static void nbif_v6_3_1_enable_doorbell_aperture(struct amdgpu_device *adev,
@@ -282,41 +260,25 @@ nbif_v6_3_1_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
 static void nbif_v6_3_1_ih_doorbell_range(struct amdgpu_device *adev,
 					  bool use_doorbell, int doorbell_index)
 {
-	u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL);
-
-	if (use_doorbell) {
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_ENABLE,
-						  0x1);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_AWID,
-						  0x0);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_OFFSET,
-						  doorbell_index);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  8);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
-						  0x0);
-	} else
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  0);
-
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_nbif_4_10,
-			     ih_doorbell_range);
-	else
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-			     ih_doorbell_range);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_offset = doorbell_index,
+		.range_size = 8,
+	};
+	u32 ctrl;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, NULL, NULL);
+		return;
+	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, &ctrl, NULL);
 }
 
 static void nbif_v6_3_1_ih_control(struct amdgpu_device *adev)
@@ -371,18 +333,12 @@ static u32 nbif_v6_3_1_get_hdp_flush_done_offset(struct amdgpu_device *adev)
 
 static u32 nbif_v6_3_1_get_pcie_index_offset(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
-		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX);
-	else
-		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_INDEX);
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_INDEX);
 }
 
 static u32 nbif_v6_3_1_get_pcie_data_offset(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
-		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA);
-	else
-		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_DATA);
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_DATA);
 }
 
 const struct nbio_hdp_flush_reg nbif_v6_3_1_hdp_flush_reg = {
@@ -404,6 +360,8 @@ static void nbif_v6_3_1_init_registers(struct amdgpu_device *adev)
 {
 	uint32_t data;
 
+	nbif_v6_3_1_init_doorbell_entry_registers(adev);
+
 	data = RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF2_STRAP2);
 	data &= ~RCC_DEV0_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F2_MASK;
 	WREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF2_STRAP2, data);
@@ -557,7 +515,6 @@ const struct amdgpu_nbio_funcs nbif_v6_3_1_funcs = {
 	.get_memsize = nbif_v6_3_1_get_memsize,
 	.sdma_doorbell_range = nbif_v6_3_1_sdma_doorbell_range,
 	.vcn_doorbell_range = nbif_v6_3_1_vcn_doorbell_range,
-	.vpe_doorbell_range = nbif_v6_3_1_vpe_doorbell_range,
 	.gc_doorbell_init = nbif_v6_3_1_gc_doorbell_init,
 	.enable_doorbell_aperture = nbif_v6_3_1_enable_doorbell_aperture,
 	.enable_doorbell_selfring_aperture = nbif_v6_3_1_enable_doorbell_selfring_aperture,
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
index a66e3073aed8..be98fee6fbf2 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
@@ -28,6 +28,81 @@
 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
 #include <uapi/linux/kfd_ioctl.h>
 
+static const struct amdgpu_nbio_doorbell_layout
+nbio_v4_3_doorbell_layout = {
+	.ctrl = {
+		.enable = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_ENABLE),
+		.awid = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWID),
+		.fence_enable = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_FENCE_ENABLE),
+		.range_offset = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_OFFSET),
+		.range_size = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_SIZE),
+		.disable_64bit = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS),
+		.deduct_range_offset = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET),
+		.awaddr_31_28 = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE),
+	},
+};
+
+static void nbio_v4_3_init_doorbell_entry_registers(
+	struct amdgpu_device *adev)
+{
+	struct amdgpu_nbio_doorbell_entries *entries;
+	u32 ctrl_base;
+
+	memset(adev->nbio.doorbell_entries, 0,
+	       sizeof(adev->nbio.doorbell_entries));
+	adev->nbio.doorbell_layout = &nbio_v4_3_doorbell_layout;
+
+	ctrl_base = SOC15_REG_OFFSET(NBIO, 0, regS2A_DOORBELL_ENTRY_0_CTRL);
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP];
+	entries->num_entries = 1;
+	entries->entry[0].port = 0;
+	entries->entry[0].ctrl_reg = ctrl_base;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH];
+	entries->num_entries = 1;
+	entries->entry[0].port = 1;
+	entries->entry[0].ctrl_reg = ctrl_base + 1;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA];
+	entries->num_entries = 1;
+	entries->entry[0].port = 2;
+	entries->entry[0].ctrl_reg = ctrl_base + 2;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_RLC];
+	entries->num_entries = 1;
+	entries->entry[0].port = 3;
+	entries->entry[0].ctrl_reg = ctrl_base + 3;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN];
+	entries->num_entries = 2;
+	entries->entry[0].port = 4;
+	entries->entry[0].ctrl_reg = ctrl_base + 4;
+	entries->entry[1].port = 5;
+	entries->entry[1].ctrl_reg = ctrl_base + 5;
+}
+
 static void nbio_v4_3_remap_hdp_registers(struct amdgpu_device *adev)
 {
 	WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL,
@@ -65,87 +140,86 @@ static void nbio_v4_3_sdma_doorbell_range(struct amdgpu_device *adev, int instan
 					  bool use_doorbell, int doorbell_index,
 					  int doorbell_size)
 {
-	if (instance == 0) {
-		u32 doorbell_range = RREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_2_CTRL);
-
-		if (use_doorbell) {
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_ENABLE,
-						       0x1);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_AWID,
-						       0xe);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_OFFSET,
-						       doorbell_index);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_SIZE,
-						       doorbell_size);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE,
-						       0x3);
-		} else
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_SIZE,
-						       0);
-
-		WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_2_CTRL, doorbell_range);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0xe,
+		.range_offset = doorbell_index,
+		.range_size = doorbell_size,
+		.awaddr_31_28 = 0x3,
+	};
+	u32 ctrl;
+
+	if (instance != 0)
+		return;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance,
+			NULL, NULL);
+		return;
 	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance, &ctrl, NULL);
 }
 
 static void nbio_v4_3_vcn_doorbell_range(struct amdgpu_device *adev, bool use_doorbell,
 					 int doorbell_index, int instance)
 {
-	u32 doorbell_range;
+	static const u8 vcn_component[] = { 0x4, 0x7 };
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_offset = doorbell_index,
+		.range_size = 8,
+	};
+	u32 ctrl;
+
+	if (WARN_ON_ONCE((unsigned int)instance >=
+			 ARRAY_SIZE(vcn_component)))
+		return;
 
-	if (instance)
-		doorbell_range = RREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_5_CTRL);
-	else
-		doorbell_range = RREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_4_CTRL);
-
-	if (use_doorbell) {
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_ENABLE,
-					       0x1);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_AWID,
-					       instance ? 0x7 : 0x4);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_RANGE_OFFSET,
-					       doorbell_index);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_RANGE_SIZE,
-					       8);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE,
-					       instance ? 0x7 : 0x4);
-	} else
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       S2A_DOORBELL_ENTRY_4_CTRL,
-					       S2A_DOORBELL_PORT4_RANGE_SIZE,
-					       0);
-
-	if (instance)
-		WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
-	else
-		WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_4_CTRL, doorbell_range);
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_VCN, instance,
+			NULL, NULL);
+		return;
+	}
+
+	fields.awid = vcn_component[instance];
+	fields.awaddr_31_28 = fields.awid;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN, instance, &ctrl, NULL);
 }
 
 static void nbio_v4_3_gc_doorbell_init(struct amdgpu_device *adev)
 {
-	WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_0_CTRL, 0x30000007);
-	WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_3_CTRL, 0x3000000d);
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awaddr_31_28 = 0x3,
+	};
+	u32 ctrl;
+
+	fields.awid = 0x3;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP, 0, &ctrl, NULL);
+
+	fields.awid = 0x6;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_RLC, 0, &ctrl, NULL);
 }
 
 static void nbio_v4_3_enable_doorbell_aperture(struct amdgpu_device *adev,
@@ -181,36 +255,25 @@ static void nbio_v4_3_enable_doorbell_selfring_aperture(struct amdgpu_device *ad
 static void nbio_v4_3_ih_doorbell_range(struct amdgpu_device *adev,
 					bool use_doorbell, int doorbell_index)
 {
-	u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_1_CTRL);
-
-	if (use_doorbell) {
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_ENABLE,
-						  0x1);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_AWID,
-						  0x0);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_OFFSET,
-						  doorbell_index);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  8);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
-						  0x0);
-	} else
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  0);
-
-	WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_offset = doorbell_index,
+		.range_size = 8,
+	};
+	u32 ctrl;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+				AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, NULL, NULL);
+		return;
+	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, &ctrl, NULL);
 }
 
 static void nbio_v4_3_ih_control(struct amdgpu_device *adev)
@@ -337,6 +400,8 @@ const struct nbio_hdp_flush_reg nbio_v4_3_hdp_flush_reg = {
 
 static void nbio_v4_3_init_registers(struct amdgpu_device *adev)
 {
+	nbio_v4_3_init_doorbell_entry_registers(adev);
+
 	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(4, 3, 0)) {
 		uint32_t data;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
index a40a13ca1011..75477951353d 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
@@ -26,6 +26,57 @@
 #include "amdgpu.h"
 #include "nbio_v6_3_2.h"
 
+static const struct amdgpu_nbio_doorbell_layout
+nbio_v6_3_2_doorbell_layout = {
+	.ctrl = {
+		.enable = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_ENABLE),
+		.awid = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWID),
+		.fence_enable = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_FENCE_ENABLE),
+		.range_offset = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_OFFSET),
+		.range_size = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_SIZE),
+		.disable_64bit = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS),
+		.deduct_range_offset = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET),
+		.drop = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_DROP_EN),
+		.awaddr_31_28 = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE),
+	},
+	.ctrl1 = {
+		.target_port_type = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1,
+			S2A_DOORBELL_PORT0_TARGET_PORT_TYPE),
+		.target_die_id = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1,
+			S2A_DOORBELL_PORT0_TARGET_DIEID),
+		.target_port_id = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1,
+			S2A_DOORBELL_PORT0_TARGET_PORT_ID),
+		.override_poison_data = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1,
+			S2A_DOORBELL_PORT0_OVERRIDE_POISON_DATA),
+		.disable_4k_page_decode = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1,
+			S2A_DOORBELL_PORT0_4K_PAGE_DECODE_DISABLE),
+	},
+	.has_ctrl1 = true,
+};
+
 static u32 nbio_v6_3_2_get_pcie_index_offset(struct amdgpu_device *adev)
 {
 	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2);
@@ -71,8 +122,57 @@ static void nbio_v6_3_2_mc_access_enable(struct amdgpu_device *adev,
 		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0);
 }
 
+static void nbio_v6_3_2_init_doorbell_entry_registers(
+	struct amdgpu_device *adev)
+{
+	struct amdgpu_nbio_doorbell_entries *entries;
+	u32 ctrl_base, ctrl1_base;
+	u32 i;
+
+	memset(adev->nbio.doorbell_entries, 0,
+	       sizeof(adev->nbio.doorbell_entries));
+	adev->nbio.doorbell_layout = &nbio_v6_3_2_doorbell_layout;
+
+	ctrl_base = SOC15_REG_OFFSET(
+		NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL);
+	ctrl1_base = SOC15_REG_OFFSET(
+		NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1);
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP];
+	entries->num_entries = 1;
+	entries->entry[0].port = 0;
+	entries->entry[0].ctrl_reg = ctrl_base;
+	entries->entry[0].ctrl1_reg = ctrl1_base;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH];
+	entries->num_entries = 1;
+	entries->entry[0].port = 1;
+	entries->entry[0].ctrl_reg = ctrl_base + 2;
+	entries->entry[0].ctrl1_reg = ctrl1_base + 2;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN];
+	entries->num_entries = 4;
+	for (i = 0; i < entries->num_entries; i++) {
+		entries->entry[i].port = i + 2;
+		entries->entry[i].ctrl_reg = ctrl_base + (i + 2) * 2;
+		entries->entry[i].ctrl1_reg = ctrl1_base + (i + 2) * 2;
+	}
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA];
+	entries->num_entries = 1;
+	entries->entry[0].port = 6;
+	entries->entry[0].ctrl_reg = ctrl_base + 6 * 2;
+	entries->entry[0].ctrl1_reg = ctrl1_base + 6 * 2;
+}
+
 static void nbio_v6_3_2_init_registers(struct amdgpu_device *adev)
 {
+	nbio_v6_3_2_init_doorbell_entry_registers(adev);
+
 	WREG32_SOC15(NBIO, 0, regXCD_DOORBELL_FENCE_1,
 		(0xff & ~(adev->gfx.xcc_mask)) <<
 		XCD_DOORBELL_FENCE_1__XCD_0_DOORBELL_DISABLE__SHIFT);
@@ -161,52 +261,44 @@ static void nbio_v6_3_2_ih_control(struct amdgpu_device *adev)
 static void nbio_v6_3_2_ih_doorbell_range(struct amdgpu_device *adev,
 					  bool use_doorbell, int doorbell_index)
 {
-	u32 ih_doorbell_range = 0;
-	u32 ih_doorbell_range1 = 0;
-
-	if (use_doorbell) {
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_ENABLE,
-						  0x1);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_AWID,
-						  0x0);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_OFFSET,
-						  doorbell_index);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  8);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
-						  0x0);
-		ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
-						   GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
-						   S2A_DOORBELL_PORT1_TARGET_PORT_TYPE,
-						   0x3);
-		ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
-						   GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
-						   S2A_DOORBELL_PORT1_TARGET_DIEID,
-						   0x0);
-		ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
-						   GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
-						   S2A_DOORBELL_PORT1_TARGET_PORT_ID,
-						   0x0);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_offset = doorbell_index,
+		.range_size = 8,
+		.target_port_type = 0x3,
+	};
+	u32 ctrl, ctrl1;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, NULL, NULL);
+		return;
 	}
 
-	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range);
-	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1, ih_doorbell_range1);
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, &ctrl1)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, &ctrl, &ctrl1);
 }
 
 static void nbio_v6_3_2_gc_doorbell_init(struct amdgpu_device *adev)
 {
-	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL, 0x30000007);
-	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1, 0x3);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0x3,
+		.awaddr_31_28 = 0x3,
+		.target_port_type = 0x3,
+	};
+	u32 ctrl, ctrl1;
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, &ctrl1)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP, 0, &ctrl, &ctrl1);
 }
 
 static void nbio_v6_3_2_sdma_doorbell_range(struct amdgpu_device *adev,
@@ -214,114 +306,68 @@ static void nbio_v6_3_2_sdma_doorbell_range(struct amdgpu_device *adev,
 					    int doorbell_index,
 					    int doorbell_size)
 {
-	if (instance == 0) {
-		u32 doorbell_range = 0;
-		u32 doorbell_range1 = 0;
-
-		if (use_doorbell) {
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
-						       S2A_DOORBELL_PORT6_ENABLE,
-						       0x1);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
-						       S2A_DOORBELL_PORT6_AWID,
-						       0xe);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
-						       S2A_DOORBELL_PORT6_RANGE_OFFSET,
-						       doorbell_index);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
-						       S2A_DOORBELL_PORT6_RANGE_SIZE,
-						       doorbell_size);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
-						       S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE,
-						       0xe);
-			doorbell_range1 = REG_SET_FIELD(doorbell_range1,
-						        GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
-						        S2A_DOORBELL_PORT6_TARGET_PORT_TYPE,
-						        0x3);
-			doorbell_range1 = REG_SET_FIELD(doorbell_range1,
-						        GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
-						        S2A_DOORBELL_PORT6_TARGET_DIEID,
-						        0x0);
-			doorbell_range1 = REG_SET_FIELD(doorbell_range1,
-						        GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
-						        S2A_DOORBELL_PORT6_TARGET_PORT_ID,
-						        0x0);
-		}
-
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, doorbell_range);
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1, doorbell_range1);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0xe,
+		.range_offset = doorbell_index,
+		.range_size = doorbell_size,
+		.awaddr_31_28 = 0xe,
+		.target_port_type = 0x3,
+	};
+	u32 ctrl, ctrl1;
+
+	if (instance != 0)
+		return;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance,
+			NULL, NULL);
+		return;
 	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, &ctrl1)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance, &ctrl, &ctrl1);
 }
 
 static void nbio_v6_3_2_vcn_doorbell_range(struct amdgpu_device *adev,
 					   bool use_doorbell, int doorbell_index,
 					   int instance)
 {
-	u32 doorbell_range = 0;
-	u32 doorbell_range1 = 0;
-
-	if (use_doorbell) {
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-					       S2A_DOORBELL_PORT2_ENABLE,
-					       0x1);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-					       S2A_DOORBELL_PORT2_AWID,
-					       (instance % adev->vcn.num_inst_per_aid) ? 0x7 : 0x4);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-					       S2A_DOORBELL_PORT2_RANGE_OFFSET,
-					       doorbell_index);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-					       S2A_DOORBELL_PORT2_RANGE_SIZE,
-					       11);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-					       S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE,
-					       (instance % adev->vcn.num_inst_per_aid) ? 0x7 : 0x4);
-		doorbell_range1 = REG_SET_FIELD(doorbell_range1,
-						GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
-					        S2A_DOORBELL_PORT2_TARGET_PORT_TYPE,
-					        0x3);
-		doorbell_range1 = REG_SET_FIELD(doorbell_range1,
-						GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
-					        S2A_DOORBELL_PORT2_TARGET_DIEID,
-					        (instance / adev->vcn.num_inst_per_aid) ? 0x3 : 0x0);
-		doorbell_range1 = REG_SET_FIELD(doorbell_range1,
-						GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
-					        S2A_DOORBELL_PORT2_TARGET_PORT_ID,
-					        0x0);
+	static const u8 vcn_component[] = { 0x4, 0x7, 0x4, 0x7 };
+	static const u8 vcn_target_die[] = { 0, 0, 3, 3 };
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_offset = doorbell_index,
+		.range_size = 11,
+		.target_port_type = 0x3,
+	};
+	u32 ctrl, ctrl1;
+
+	if (WARN_ON_ONCE((unsigned int)instance >=
+			 ARRAY_SIZE(vcn_component)))
+		return;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_VCN, instance,
+			NULL, NULL);
+		return;
 	}
 
-	switch (instance) {
-	case 0:
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, doorbell_range);
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1, doorbell_range1);
-		break;
-	case 1:
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL, doorbell_range);
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1, doorbell_range1);
-		break;
-	case 2:
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL, doorbell_range);
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1, doorbell_range1);
-		break;
-	case 3:
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1, doorbell_range1);
-		break;
-	default:
-		dev_err(adev->dev,
-			"amdgpu: invalid vcn instance set when program doorbell range\n");
-		break;
-	}
+	fields.awid = vcn_component[instance];
+	fields.awaddr_31_28 = fields.awid;
+	fields.target_die_id = vcn_target_die[instance];
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, &ctrl1)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN, instance, &ctrl, &ctrl1);
 }
 
 static int nbio_v6_3_2_get_compute_partition_mode(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_4.c
new file mode 100644
index 000000000000..9e60613addc8
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_4.c
@@ -0,0 +1,452 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "nbio_v7_11_4.h"
+
+#include "nbif/nbif_4_10_0_offset.h"
+#include "nbif/nbif_4_10_0_sh_mask.h"
+#include <uapi/linux/kfd_ioctl.h>
+
+static const struct amdgpu_nbio_doorbell_layout
+nbio_v7_11_4_doorbell_layout = {
+	.ctrl = {
+		.enable = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_ENABLE),
+		.awid = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWID),
+		.fence_enable = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_FENCE_ENABLE),
+		.range_offset = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_OFFSET),
+		.range_size = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_SIZE),
+		.disable_64bit = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS),
+		.deduct_range_offset = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET),
+		.drop = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_DROP_EN),
+		.awaddr_31_28 = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE),
+	},
+};
+
+static void nbio_v7_11_4_init_doorbell_entry_registers(
+	struct amdgpu_device *adev)
+{
+	struct amdgpu_nbio_doorbell_entries *entries;
+	u32 ctrl_base;
+
+	memset(adev->nbio.doorbell_entries, 0,
+	       sizeof(adev->nbio.doorbell_entries));
+	adev->nbio.doorbell_layout = &nbio_v7_11_4_doorbell_layout;
+
+	ctrl_base = SOC15_REG_OFFSET(
+		NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL);
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP];
+	entries->num_entries = 1;
+	entries->entry[0].port = 0;
+	entries->entry[0].ctrl_reg = ctrl_base;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH];
+	entries->num_entries = 1;
+	entries->entry[0].port = 1;
+	entries->entry[0].ctrl_reg = ctrl_base + 2;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA];
+	entries->num_entries = 1;
+	entries->entry[0].port = 2;
+	entries->entry[0].ctrl_reg = ctrl_base + 4;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_RLC];
+	entries->num_entries = 1;
+	entries->entry[0].port = 3;
+	entries->entry[0].ctrl_reg = ctrl_base + 6;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN];
+	entries->num_entries = 1;
+	entries->entry[0].port = 4;
+	entries->entry[0].ctrl_reg = ctrl_base + 8;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_VPE];
+	entries->num_entries = 1;
+	entries->entry[0].port = 5;
+	entries->entry[0].ctrl_reg = ctrl_base + 10;
+}
+
+static void nbio_v7_11_4_remap_hdp_registers(struct amdgpu_device *adev)
+{
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL,
+		adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL,
+		adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
+}
+
+static u32 nbio_v7_11_4_get_rev_id(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0);
+	tmp &= RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK;
+	tmp >>= RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT;
+
+	return tmp;
+}
+
+static void nbio_v7_11_4_mc_access_enable(struct amdgpu_device *adev, bool enable)
+{
+	if (enable)
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN,
+			     BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK |
+			     BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK);
+	else
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0);
+}
+
+static u32 nbio_v7_11_4_get_memsize(struct amdgpu_device *adev)
+{
+	return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE);
+}
+
+static void nbio_v7_11_4_sdma_doorbell_range(struct amdgpu_device *adev,
+					    int instance, bool use_doorbell,
+					    int doorbell_index,
+					    int doorbell_size)
+{
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0xe,
+		.range_offset = doorbell_index,
+		.range_size = doorbell_size,
+		.awaddr_31_28 = 0x3,
+	};
+	u32 ctrl;
+
+	if (instance != 0)
+		return;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance,
+			NULL, NULL);
+		return;
+	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance, &ctrl, NULL);
+}
+
+static void nbio_v7_11_4_vcn_doorbell_range(struct amdgpu_device *adev,
+					   bool use_doorbell, int doorbell_index,
+					   int instance)
+{
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0x4,
+		.range_offset = doorbell_index,
+		.range_size = 8,
+		.awaddr_31_28 = 0x4,
+	};
+	u32 ctrl;
+
+	if (instance)
+		return;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_VCN, 0,
+			NULL, NULL);
+		return;
+	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN, 0, &ctrl, NULL);
+}
+
+static void nbio_v7_11_4_vpe_doorbell_range(struct amdgpu_device *adev,
+					    int instance, bool use_doorbell,
+					    int doorbell_index,
+					    int doorbell_size)
+{
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0xf,
+		.range_offset = doorbell_index,
+		.range_size = doorbell_size,
+		.awaddr_31_28 = 0xf,
+	};
+	u32 ctrl;
+
+	if (instance)
+		return;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_VPE, instance,
+			NULL, NULL);
+		return;
+	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_VPE, instance, &ctrl, NULL);
+}
+
+static void nbio_v7_11_4_gc_doorbell_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awaddr_31_28 = 0x3,
+	};
+	u32 ctrl;
+
+	fields.awid = 0x3;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP, 0, &ctrl, NULL);
+
+	fields.awid = 0x6;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_RLC, 0, &ctrl, NULL);
+}
+
+static void nbio_v7_11_4_enable_doorbell_aperture(struct amdgpu_device *adev,
+						 bool enable)
+{
+	WREG32_FIELD15_PREREG(NBIO, 0, RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN,
+			BIF_DOORBELL_APER_EN, enable ? 1 : 0);
+}
+
+static void
+nbio_v7_11_4_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
+					      bool enable)
+{
+	u32 tmp = 0;
+
+	if (enable) {
+		tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_EN, 1) |
+		      REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_MODE, 1) |
+		      REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_SIZE, 0);
+
+		WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW,
+			     lower_32_bits(adev->doorbell.base));
+		WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH,
+			     upper_32_bits(adev->doorbell.base));
+	}
+
+	WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, tmp);
+}
+
+static void nbio_v7_11_4_ih_doorbell_range(struct amdgpu_device *adev,
+					  bool use_doorbell, int doorbell_index)
+{
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_offset = doorbell_index,
+		.range_size = 2,
+	};
+	u32 ctrl;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, NULL, NULL);
+		return;
+	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, &ctrl, NULL);
+}
+
+static void nbio_v7_11_4_ih_control(struct amdgpu_device *adev)
+{
+	u32 interrupt_cntl;
+
+	/* setup interrupt control */
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
+
+	interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL);
+	/*
+	 * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
+	 * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
+	 */
+	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL,
+				       IH_DUMMY_RD_OVERRIDE, 0);
+
+	/* BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
+	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL,
+				       IH_REQ_NONSNOOP_EN, 0);
+
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl);
+}
+
+static void
+nbio_v7_11_4_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+					     bool enable)
+{
+}
+
+static void
+nbio_v7_11_4_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+					    bool enable)
+{
+}
+
+static void
+nbio_v7_11_4_get_clockgating_state(struct amdgpu_device *adev,
+				  u64 *flags)
+{
+}
+
+static u32 nbio_v7_11_4_get_hdp_flush_req_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ);
+}
+
+static u32 nbio_v7_11_4_get_hdp_flush_done_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE);
+}
+
+static u32 nbio_v7_11_4_get_pcie_index_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX);
+}
+
+static u32 nbio_v7_11_4_get_pcie_data_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA);
+}
+
+const struct nbio_hdp_flush_reg nbio_v7_11_4_hdp_flush_reg = {
+	.ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK,
+	.ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK,
+	.ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK,
+	.ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK,
+	.ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK,
+	.ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK,
+	.ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK,
+	.ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK,
+	.ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK,
+	.ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK,
+	.ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK,
+	.ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK,
+};
+
+static void nbio_v7_11_4_init_registers(struct amdgpu_device *adev)
+{
+	uint32_t data;
+
+	nbio_v7_11_4_init_doorbell_entry_registers(adev);
+
+	data = RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF2_STRAP2);
+	data &= ~RCC_DEV0_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F2_MASK;
+	WREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF2_STRAP2, data);
+}
+
+static u32 nbio_v7_11_4_get_rom_offset(struct amdgpu_device *adev)
+{
+	u32 data, rom_offset;
+
+	data = RREG32_SOC15(NBIO, 0, regREGS_ROM_OFFSET_CTRL);
+	rom_offset = REG_GET_FIELD(data, REGS_ROM_OFFSET_CTRL, ROM_OFFSET);
+
+	return rom_offset;
+}
+
+#define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE)
+
+static void nbio_v7_11_4_set_reg_remap(struct amdgpu_device *adev)
+{
+	if (!amdgpu_sriov_vf(adev) && (PAGE_SIZE <= 4096)) {
+		adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
+		adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
+	} else {
+		adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
+			regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
+		adev->rmmio_remap.bus_addr = 0;
+	}
+}
+
+const struct amdgpu_nbio_funcs nbio_v7_11_4_funcs = {
+	.get_hdp_flush_req_offset = nbio_v7_11_4_get_hdp_flush_req_offset,
+	.get_hdp_flush_done_offset = nbio_v7_11_4_get_hdp_flush_done_offset,
+	.get_pcie_index_offset = nbio_v7_11_4_get_pcie_index_offset,
+	.get_pcie_data_offset = nbio_v7_11_4_get_pcie_data_offset,
+	.get_rev_id = nbio_v7_11_4_get_rev_id,
+	.mc_access_enable = nbio_v7_11_4_mc_access_enable,
+	.get_memsize = nbio_v7_11_4_get_memsize,
+	.sdma_doorbell_range = nbio_v7_11_4_sdma_doorbell_range,
+	.vcn_doorbell_range = nbio_v7_11_4_vcn_doorbell_range,
+	.vpe_doorbell_range = nbio_v7_11_4_vpe_doorbell_range,
+	.gc_doorbell_init = nbio_v7_11_4_gc_doorbell_init,
+	.enable_doorbell_aperture = nbio_v7_11_4_enable_doorbell_aperture,
+	.enable_doorbell_selfring_aperture = nbio_v7_11_4_enable_doorbell_selfring_aperture,
+	.ih_doorbell_range = nbio_v7_11_4_ih_doorbell_range,
+	.update_medium_grain_clock_gating = nbio_v7_11_4_update_medium_grain_clock_gating,
+	.update_medium_grain_light_sleep = nbio_v7_11_4_update_medium_grain_light_sleep,
+	.get_clockgating_state = nbio_v7_11_4_get_clockgating_state,
+	.ih_control = nbio_v7_11_4_ih_control,
+	.init_registers = nbio_v7_11_4_init_registers,
+	.remap_hdp_registers = nbio_v7_11_4_remap_hdp_registers,
+	.get_rom_offset = nbio_v7_11_4_get_rom_offset,
+	.set_reg_remap = nbio_v7_11_4_set_reg_remap,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_4.h b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_4.h
new file mode 100644
index 000000000000..2757fb8ccfa9
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_4.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __NBIO_V7_11_4_H__
+#define __NBIO_V7_11_4_H__
+
+#include "soc15_common.h"
+
+extern const struct nbio_hdp_flush_reg nbio_v7_11_4_hdp_flush_reg;
+extern const struct amdgpu_nbio_funcs nbio_v7_11_4_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c
index 0ea6e091a24c..47db8f87d72f 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c
@@ -28,6 +28,83 @@
 #include "nbio/nbio_7_11_5_sh_mask.h"
 #include <uapi/linux/kfd_ioctl.h>
 
+static const struct amdgpu_nbio_doorbell_layout
+nbio_v7_11_5_doorbell_layout = {
+	.ctrl = {
+		.enable = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_ENABLE),
+		.awid = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWID),
+		.fence_enable = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_FENCE_ENABLE),
+		.range_offset = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_OFFSET),
+		.range_size = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_SIZE),
+		.disable_64bit = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS),
+		.deduct_range_offset = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET),
+		.drop = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_DROP_EN),
+		.awaddr_31_28 = REG_FIELD_MASK(
+			GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE),
+	},
+};
+
+static void nbio_v7_11_5_init_doorbell_entry_registers(
+	struct amdgpu_device *adev)
+{
+	struct amdgpu_nbio_doorbell_entries *entries;
+	u32 ctrl_base;
+
+	memset(adev->nbio.doorbell_entries, 0,
+	       sizeof(adev->nbio.doorbell_entries));
+	adev->nbio.doorbell_layout = &nbio_v7_11_5_doorbell_layout;
+
+	ctrl_base = SOC15_REG_OFFSET(
+		NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL);
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP];
+	entries->num_entries = 1;
+	entries->entry[0].port = 0;
+	entries->entry[0].ctrl_reg = ctrl_base;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH];
+	entries->num_entries = 1;
+	entries->entry[0].port = 1;
+	entries->entry[0].ctrl_reg = ctrl_base + 2;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA];
+	entries->num_entries = 1;
+	entries->entry[0].port = 2;
+	entries->entry[0].ctrl_reg = ctrl_base + 4;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_RLC];
+	entries->num_entries = 1;
+	entries->entry[0].port = 3;
+	entries->entry[0].ctrl_reg = ctrl_base + 6;
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_VPE];
+	entries->num_entries = 1;
+	entries->entry[0].port = 5;
+	entries->entry[0].ctrl_reg = ctrl_base + 10;
+}
+
 static void nbio_v7_11_5_remap_hdp_registers(struct amdgpu_device *adev)
 {
 		WREG32_SOC15(NBIO, 0, regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL,
@@ -72,42 +149,31 @@ static void nbio_v7_11_5_sdma_doorbell_range(struct amdgpu_device *adev,
 					    int doorbell_index,
 					    int doorbell_size)
 {
-	u32 doorbell_range;
-
-	if (instance == 0) {
-		doorbell_range = RREG32_SOC15(NBIO, 0,
-					     regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL);
-
-		if (use_doorbell) {
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_ENABLE,
-						       0x1);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_AWID,
-						       0xe);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_OFFSET,
-						       doorbell_index);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_SIZE,
-						       doorbell_size);
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE,
-						       0x3);
-		} else {
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
-						       S2A_DOORBELL_PORT2_RANGE_SIZE,
-						       0);
-		}
-
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, doorbell_range);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0xe,
+		.range_offset = doorbell_index,
+		.range_size = doorbell_size,
+		.awaddr_31_28 = 0x3,
+	};
+	u32 ctrl;
+
+	if (instance != 0)
+		return;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance,
+			NULL, NULL);
+		return;
 	}
+
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance, &ctrl, NULL);
 }
 
 static void nbio_v7_11_5_vpe_doorbell_range(struct amdgpu_device *adev,
@@ -115,48 +181,54 @@ static void nbio_v7_11_5_vpe_doorbell_range(struct amdgpu_device *adev,
 							int doorbell_index,
 							int doorbell_size)
 {
-	u32 doorbell_range;
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0xf,
+		.range_offset = doorbell_index,
+		.range_size = doorbell_size,
+		.awaddr_31_28 = 0xf,
+	};
+	u32 ctrl;
 
 	if (instance)
 		return;
 
-	doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL);
-
-	if (use_doorbell) {
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_ENABLE,
-					       0x1);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_AWID,
-					       0xf);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_RANGE_OFFSET,
-					       doorbell_index);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_RANGE_SIZE,
-					       doorbell_size);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE,
-					       0xf);
-	} else {
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
-					       S2A_DOORBELL_PORT5_RANGE_SIZE,
-					       0);
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_VPE, instance,
+			NULL, NULL);
+		return;
 	}
 
-	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_VPE, instance, &ctrl, NULL);
 }
 
 static void nbio_v7_11_5_gc_doorbell_init(struct amdgpu_device *adev)
 {
-	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL, 0x30000007);
-	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL, 0x3000000d);
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awaddr_31_28 = 0x3,
+	};
+	u32 ctrl;
+
+	fields.awid = 0x3;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_CP, 0, &ctrl, NULL);
+
+	fields.awid = 0x6;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_RLC, 0, &ctrl, NULL);
 }
 
 static void nbio_v7_11_5_enable_doorbell_aperture(struct amdgpu_device *adev,
@@ -191,39 +263,25 @@ static void nbio_v7_11_5_enable_doorbell_selfring_aperture(struct amdgpu_device
 static void nbio_v7_11_5_ih_doorbell_range(struct amdgpu_device *adev,
 					 bool use_doorbell, int doorbell_index)
 {
-	u32 ih_doorbell_range;
-
-	ih_doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL);
-
-	if (use_doorbell) {
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_ENABLE,
-						  0x1);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_AWID,
-						  0x0);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_OFFSET,
-						  doorbell_index);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  2);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
-						  0x0);
-	} else {
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
-						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  0);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_offset = doorbell_index,
+		.range_size = 2,
+	};
+	u32 ctrl;
+
+	if (!use_doorbell) {
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, NULL, NULL);
+		return;
 	}
 
-	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range);
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+				adev, &fields, &ctrl, NULL)))
+		return;
+
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, &ctrl, NULL);
 }
 
 static void nbio_v7_11_5_ih_control(struct amdgpu_device *adev)
@@ -310,6 +368,8 @@ static void nbio_v7_11_5_init_registers(struct amdgpu_device *adev)
 {
 	uint32_t data;
 
+	nbio_v7_11_5_init_doorbell_entry_registers(adev);
+
 	data = RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF2_STRAP2);
 	data &= ~RCC_DEV0_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F2_MASK;
 	WREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF2_STRAP2, data);
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
index ad24441776d2..21dda3401b4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
@@ -32,6 +32,131 @@
 
 #define NPS_MODE_MASK 0x000000FFL
 
+static const struct amdgpu_nbio_doorbell_layout
+nbio_v7_9_doorbell_layout = {
+	.ctrl = {
+		.enable = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_ENABLE),
+		.awid = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWID),
+		.fence_enable = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_FENCE_ENABLE),
+		.range_offset = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_OFFSET),
+		.range_size = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_RANGE_SIZE),
+		.disable_64bit = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS),
+		.deduct_range_offset = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET),
+		.awaddr_31_28 = REG_FIELD_MASK(
+			S2A_DOORBELL_ENTRY_0_CTRL,
+			S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE),
+	},
+};
+
+static void nbio_v7_9_init_doorbell_entry_registers(
+	struct amdgpu_device *adev)
+{
+	static const u8 sdma_port[] = { 1, 2, 5, 6 };
+	struct amdgpu_nbio_doorbell_entries *entries;
+	u32 ctrl_base, dev_inst, local_inst;
+	int i;
+
+	memset(adev->nbio.doorbell_entries, 0,
+	       sizeof(adev->nbio.doorbell_entries));
+	adev->nbio.doorbell_layout = &nbio_v7_9_doorbell_layout;
+
+	ctrl_base = SOC15_REG_OFFSET(NBIO, 0,
+				     regS2A_DOORBELL_ENTRY_0_CTRL);
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH];
+	entries->num_entries = 1;
+	entries->entry[0].port = 3;
+	entries->entry[0].ctrl_reg = ctrl_base + 3;
+
+	/*
+	 * Unlike IH, SDMA and VCN use global SMN addresses. Their callers invoke
+	 * the callbacks once per available instance to cover all AIDs across the
+	 * die.
+	 */
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA];
+	entries->num_entries = adev->sdma.num_instances;
+	if (WARN_ON_ONCE(entries->num_entries >
+			 AMDGPU_NBIO_MAX_DOORBELL_INSTANCES) ||
+	    WARN_ON_ONCE(entries->num_entries &&
+			 !adev->sdma.num_inst_per_aid)) {
+		entries->num_entries = 0;
+		return;
+	}
+	for (i = 0; i < entries->num_entries; i++) {
+		dev_inst = GET_INST(SDMA0, i);
+		local_inst = dev_inst % adev->sdma.num_inst_per_aid;
+		if (WARN_ON_ONCE(local_inst >= ARRAY_SIZE(sdma_port))) {
+			entries->num_entries = 0;
+			return;
+		}
+		entries->entry[i].port = sdma_port[local_inst];
+		entries->entry[i].ctrl_reg =
+			regS2A_DOORBELL_ENTRY_0_CTRL + sdma_port[local_inst];
+		entries->entry[i].use_ext = true;
+		entries->entry[i].reg_inst = adev->sdma.instance[i].aid_id;
+		entries->entry[i].ctrl_reg_base_idx =
+			regS2A_DOORBELL_ENTRY_0_CTRL_BASE_IDX;
+	}
+
+	entries = &adev->nbio.doorbell_entries[
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN];
+	entries->num_entries = fls(adev->aid_mask);
+	if (WARN_ON_ONCE(entries->num_entries >
+			 AMDGPU_NBIO_MAX_DOORBELL_INSTANCES)) {
+		entries->num_entries = 0;
+		return;
+	}
+	for_each_inst(i, adev->aid_mask) {
+		entries->entry[i].port = 4;
+		entries->entry[i].ctrl_reg = regS2A_DOORBELL_ENTRY_4_CTRL;
+		entries->entry[i].use_ext = true;
+		entries->entry[i].reg_inst = i;
+		entries->entry[i].ctrl_reg_base_idx =
+			regS2A_DOORBELL_ENTRY_4_CTRL_BASE_IDX;
+	}
+}
+
+static u32 nbio_v7_9_build_doorbell_range(u32 range_offset,
+					  u32 range_size, u32 fence_enable)
+{
+	u32 doorbell_range = 0;
+
+	doorbell_range = REG_SET_FIELD(doorbell_range,
+		DOORBELL0_CTRL_ENTRY_0, BIF_DOORBELL0_RANGE_OFFSET_ENTRY,
+		range_offset);
+	doorbell_range = REG_SET_FIELD(doorbell_range,
+		DOORBELL0_CTRL_ENTRY_0, BIF_DOORBELL0_RANGE_SIZE_ENTRY,
+		range_size);
+	doorbell_range = REG_SET_FIELD(doorbell_range,
+		DOORBELL0_CTRL_ENTRY_0, DOORBELL0_FENCE_ENABLE_ENTRY,
+		fence_enable);
+
+	return doorbell_range;
+}
+
+static void nbio_v7_9_program_doorbell_range(struct amdgpu_device *adev,
+					      u32 entry, u32 doorbell_range)
+{
+	WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_0,
+			     entry, doorbell_range);
+}
+
 static void nbio_v7_9_remap_hdp_registers(struct amdgpu_device *adev)
 {
 	WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL,
@@ -76,160 +201,80 @@ static u32 nbio_v7_9_get_memsize(struct amdgpu_device *adev)
 static void nbio_v7_9_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
 			bool use_doorbell, int doorbell_index, int doorbell_size)
 {
-	u32 doorbell_range = 0, doorbell_ctrl = 0;
-	int aid_id, dev_inst;
+	static const u8 sdma_awid[] = { 0xe, 0x8, 0x9, 0xa };
+	static const u8 sdma_awaddr[] = { 0x1, 0x2, 0x8, 0x9 };
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_size = doorbell_size,
+	};
+	u32 doorbell_range, ctrl;
+	int aid_id, dev_inst, local_inst;
 
 	dev_inst = GET_INST(SDMA0, instance);
 	aid_id = adev->sdma.instance[instance].aid_id;
+	local_inst = dev_inst % adev->sdma.num_inst_per_aid;
 
-	if (use_doorbell == false)
+	if (WARN_ON_ONCE((unsigned int)local_inst >= ARRAY_SIZE(sdma_awid)))
 		return;
 
-	doorbell_range =
-		REG_SET_FIELD(doorbell_range, DOORBELL0_CTRL_ENTRY_0,
-			BIF_DOORBELL0_RANGE_OFFSET_ENTRY, doorbell_index);
-	doorbell_range =
-		REG_SET_FIELD(doorbell_range, DOORBELL0_CTRL_ENTRY_0,
-			BIF_DOORBELL0_RANGE_SIZE_ENTRY, doorbell_size);
-	doorbell_ctrl =
-		REG_SET_FIELD(doorbell_ctrl, S2A_DOORBELL_ENTRY_1_CTRL,
-			S2A_DOORBELL_PORT1_ENABLE, 1);
-	doorbell_ctrl =
-		REG_SET_FIELD(doorbell_ctrl, S2A_DOORBELL_ENTRY_1_CTRL,
-			S2A_DOORBELL_PORT1_RANGE_SIZE, doorbell_size);
-
-	switch (dev_inst % adev->sdma.num_inst_per_aid) {
-	case 0:
-		WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_1,
-			4 * aid_id, doorbell_range);
-
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_AWID, 0xe);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_RANGE_OFFSET, 0xe);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
-					0x1);
-		WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_1_CTRL,
-			aid_id, doorbell_ctrl);
-		break;
-	case 1:
-		WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_2,
-			4 * aid_id, doorbell_range);
-
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_AWID, 0x8);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x8);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
-					0x2);
-		WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_2_CTRL,
-			aid_id, doorbell_ctrl);
-		break;
-	case 2:
-		WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_3,
-			4 * aid_id, doorbell_range);
-
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_AWID, 0x9);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x9);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
-					0x8);
-		WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_5_CTRL,
-			aid_id, doorbell_ctrl);
-		break;
-	case 3:
-		WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_4,
-			4 * aid_id, doorbell_range);
-
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_AWID, 0xa);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_RANGE_OFFSET, 0xa);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-					S2A_DOORBELL_ENTRY_1_CTRL,
-					S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
-					0x9);
-		WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_6_CTRL,
-			aid_id, doorbell_ctrl);
-		break;
-	default:
-		break;
+	if (!use_doorbell) {
+		nbio_v7_9_program_doorbell_range(adev,
+			1 + local_inst + 4 * aid_id, 0);
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance,
+			NULL, NULL);
+		return;
 	}
+
+	doorbell_range = nbio_v7_9_build_doorbell_range(
+		doorbell_index, doorbell_size, 0);
+	fields.awid = sdma_awid[local_inst];
+	fields.range_offset = fields.awid;
+	fields.awaddr_31_28 = sdma_awaddr[local_inst];
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+			adev, &fields, &ctrl, NULL)))
+		return;
+
+	nbio_v7_9_program_doorbell_range(adev,
+		1 + local_inst + 4 * aid_id, doorbell_range);
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_SDMA, instance, &ctrl, NULL);
 }
 
 static void nbio_v7_9_vcn_doorbell_range(struct amdgpu_device *adev, bool use_doorbell,
 					 int doorbell_index, int instance)
 {
-	u32 doorbell_range = 0, doorbell_ctrl = 0;
+	struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.awid = 0x4,
+		.range_offset = 0x4,
+		.awaddr_31_28 = 0x4,
+	};
+	u32 doorbell_range, ctrl;
 	u32 aid_id = instance;
 	u32 range_size;
 
-	if (use_doorbell) {
-		range_size = (amdgpu_ip_version(adev, GC_HWIP, 0) ==
+	if (!use_doorbell) {
+		nbio_v7_9_program_doorbell_range(adev, 17 + aid_id, 0);
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_VCN, aid_id,
+			NULL, NULL);
+		return;
+	}
+
+	range_size = (amdgpu_ip_version(adev, GC_HWIP, 0) ==
 						IP_VERSION(9, 5, 0)) ?
 						0xb : 0x9;
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-				DOORBELL0_CTRL_ENTRY_0,
-				BIF_DOORBELL0_RANGE_OFFSET_ENTRY,
-				doorbell_index);
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-				DOORBELL0_CTRL_ENTRY_0,
-				BIF_DOORBELL0_RANGE_SIZE_ENTRY,
-				range_size);
-		if (aid_id)
-			doorbell_range = REG_SET_FIELD(doorbell_range,
-					DOORBELL0_CTRL_ENTRY_0,
-					DOORBELL0_FENCE_ENABLE_ENTRY,
-					0x4);
-
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_ENABLE, 1);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_AWID, 0x4);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x4);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_RANGE_SIZE, range_size);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, 0x4);
-
-		WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_17,
-					aid_id, doorbell_range);
-		WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_4_CTRL,
-				aid_id, doorbell_ctrl);
-	} else {
-		doorbell_range = REG_SET_FIELD(doorbell_range,
-				DOORBELL0_CTRL_ENTRY_0,
-				BIF_DOORBELL0_RANGE_SIZE_ENTRY, 0);
-		doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_RANGE_SIZE, 0);
-
-		WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_17,
-					aid_id, doorbell_range);
-		WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_4_CTRL,
-				aid_id, doorbell_ctrl);
-	}
+	doorbell_range = nbio_v7_9_build_doorbell_range(
+		doorbell_index, range_size, aid_id ? 0x4 : 0);
+	fields.range_size = range_size;
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+			adev, &fields, &ctrl, NULL)))
+		return;
+
+	nbio_v7_9_program_doorbell_range(adev, 17 + aid_id, doorbell_range);
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_VCN, aid_id, &ctrl, NULL);
 }
 
 static void nbio_v7_9_enable_doorbell_aperture(struct amdgpu_device *adev,
@@ -266,44 +311,28 @@ static void nbio_v7_9_enable_doorbell_selfring_aperture(struct amdgpu_device *ad
 static void nbio_v7_9_ih_doorbell_range(struct amdgpu_device *adev,
 					bool use_doorbell, int doorbell_index)
 {
-	u32 ih_doorbell_range = 0, ih_doorbell_ctrl = 0;
-
-	if (use_doorbell) {
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-				DOORBELL0_CTRL_ENTRY_0,
-				BIF_DOORBELL0_RANGE_OFFSET_ENTRY,
-				doorbell_index);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-				DOORBELL0_CTRL_ENTRY_0,
-				BIF_DOORBELL0_RANGE_SIZE_ENTRY,
-				0x8);
-
-		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_ENABLE, 1);
-		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_AWID, 0);
-		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_RANGE_OFFSET, 0);
-		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_RANGE_SIZE, 0x8);
-		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, 0);
-	} else {
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
-				DOORBELL0_CTRL_ENTRY_0,
-				BIF_DOORBELL0_RANGE_SIZE_ENTRY, 0);
-		ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl,
-				S2A_DOORBELL_ENTRY_1_CTRL,
-				S2A_DOORBELL_PORT1_RANGE_SIZE, 0);
+	const struct amdgpu_nbio_doorbell_fields fields = {
+		.enable = true,
+		.range_size = 0x8,
+	};
+	u32 doorbell_range, ctrl;
+
+	if (!use_doorbell) {
+		nbio_v7_9_program_doorbell_range(adev, 0, 0);
+		amdgpu_nbio_program_doorbell_entry(adev,
+			AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, NULL, NULL);
+		return;
 	}
 
-	WREG32_SOC15(NBIO, 0, regDOORBELL0_CTRL_ENTRY_0, ih_doorbell_range);
-	WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_3_CTRL, ih_doorbell_ctrl);
+	doorbell_range = nbio_v7_9_build_doorbell_range(
+		doorbell_index, 0x8, 0);
+	if (WARN_ON_ONCE(amdgpu_nbio_build_doorbell_entry(
+			adev, &fields, &ctrl, NULL)))
+		return;
+
+	nbio_v7_9_program_doorbell_range(adev, 0, doorbell_range);
+	amdgpu_nbio_program_doorbell_entry(adev,
+		AMDGPU_NBIO_DOORBELL_CLIENT_IH, 0, &ctrl, NULL);
 }
 
 
@@ -436,6 +465,8 @@ static void nbio_v7_9_init_registers(struct amdgpu_device *adev)
 	u32 inst_mask;
 	int i;
 
+	nbio_v7_9_init_doorbell_entry_registers(adev);
+
 	WREG32_SOC15(NBIO, 0, regXCC_DOORBELL_FENCE,
 		0xff & ~(adev->gfx.xcc_mask));
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
index 21882f2cd2ae..bc7de4c3861f 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
@@ -795,7 +795,7 @@ static int sdma_v7_1_reset_queue(struct amdgpu_ring *ring,
 
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
-	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true, 0);
+	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, false, 0);
 	if (r)
 		return r;
 
@@ -857,47 +857,47 @@ static int sdma_v7_1_inst_start(struct amdgpu_device *adev,
 static int sdma_v7_1_mqd_init(struct amdgpu_device *adev, void *mqd,
 			      struct amdgpu_mqd_prop *prop)
 {
-	struct v12_sdma_mqd *m = mqd;
+	struct v12_1_sdma_mqd *m = mqd;
 	uint64_t wb_gpu_addr;
 
-	m->sdmax_rlcx_rb_cntl =
+	m->sdmax_queuen_rb_cntl =
 		order_base_2(prop->queue_size / 4) << SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
 		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
 		4 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
 		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT;
 
-	m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
-	m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
+	m->sdmax_queuen_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
+	m->sdmax_queuen_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
 
 	wb_gpu_addr = prop->wptr_gpu_addr;
-	m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
-	m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
+	m->sdmax_queuen_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
+	m->sdmax_queuen_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
 
 	wb_gpu_addr = prop->rptr_gpu_addr;
-	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
-	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
+	m->sdmax_queuen_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
+	m->sdmax_queuen_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
 
-	m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, 0,
+	m->sdmax_queuen_ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, 0,
 							regSDMA0_SDMA_QUEUE0_IB_CNTL));
 
-	m->sdmax_rlcx_doorbell_offset =
+	m->sdmax_queuen_doorbell_offset =
 		prop->doorbell_index << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
 
-	m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_SDMA_QUEUE0_DOORBELL, ENABLE, 1);
+	m->sdmax_queuen_doorbell = REG_SET_FIELD(0, SDMA0_SDMA_QUEUE0_DOORBELL, ENABLE, 1);
 
-	m->sdmax_rlcx_doorbell_log = 0;
-	m->sdmax_rlcx_rb_aql_cntl = 0x4000;	//regSDMA0_SDMA_QUEUE0_RB_AQL_CNTL_DEFAULT;
-	m->sdmax_rlcx_dummy_reg = 0xf;	//regSDMA0_SDMA_QUEUE0_DUMMY_REG_DEFAULT;
+	m->sdmax_queuen_doorbell_log = 0;
+	m->sdmax_queuen_rb_aql_cntl = 0x4000;	//regSDMA0_SDMA_QUEUE0_RB_AQL_CNTL_DEFAULT;
+	m->sdmax_queuen_dummy_reg = 0xf;	//regSDMA0_SDMA_QUEUE0_DUMMY_REG_DEFAULT;
 
-	m->sdmax_rlcx_csa_addr_lo = lower_32_bits(prop->csa_addr);
-	m->sdmax_rlcx_csa_addr_hi = upper_32_bits(prop->csa_addr);
+	m->sdmax_queuen_csa_addr_lo = lower_32_bits(prop->csa_addr);
+	m->sdmax_queuen_csa_addr_hi = upper_32_bits(prop->csa_addr);
 
 	return 0;
 }
 
 static void sdma_v7_1_set_mqd_funcs(struct amdgpu_device *adev)
 {
-	adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v12_sdma_mqd);
+	adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v12_1_sdma_mqd);
 	adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v7_1_mqd_init;
 }
 
@@ -1333,8 +1333,7 @@ static int sdma_v7_1_sw_init(struct amdgpu_ip_block *ip_block)
 
 	adev->sdma.supported_reset =
 		amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
-	if (!amdgpu_sriov_vf(adev) &&
-	    !adev->debug_disable_gpu_ring_reset)
+	if (!adev->debug_disable_gpu_ring_reset)
 		adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
 
 	r = amdgpu_sdma_sysfs_reset_mask_init(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 0adfd4b101aa..e5e590425f42 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -1512,6 +1512,16 @@ static void si_invalidate_hdp(struct amdgpu_device *adev,
 
 static bool si_need_reset_on_init(struct amdgpu_device *adev)
 {
+	/*
+	 * MacBookPros with switchable graphics put VRAM at 0 when the iGPU
+	 * is enabled.  Any later post (S3 resume, GPU reset) moves it back to
+	 * the vbios default, so reset and post on load to start with the
+	 * vbios layout.
+	 */
+	if (adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE &&
+	    !(RREG32(mmMC_VM_FB_LOCATION) & MC_VM_FB_LOCATION__FB_BASE_MASK))
+		return true;
+
 	return false;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index a0435fd5525b..cad6bbfcedf3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1405,6 +1405,16 @@ static bool vi_need_reset_on_init(struct amdgpu_device *adev)
 	unsigned int i;
 	u32 clock_cntl, pc;
 
+	/*
+	 * MacBookPros with switchable graphics put VRAM at 0 when the iGPU
+	 * is enabled.  Any later post (S3 resume, GPU reset) moves it back to
+	 * the vbios default, so reset and post on load to start with the
+	 * vbios layout.
+	 */
+	if (adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE &&
+	    !(RREG32(mmMC_VM_FB_LOCATION) & MC_VM_FB_LOCATION__FB_BASE_MASK))
+		return true;
+
 	for (i = 0; i < ARRAY_SIZE(vi_reset_quirks); i++)
 		if (adev->pdev->device == vi_reset_quirks[i].device &&
 		    adev->pdev->subsystem_vendor == vi_reset_quirks[i].subsystem_vendor &&
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 344da6c0e96a..5f8fb934720b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -427,12 +427,10 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 		}
 	}
 
-	if (!pdd->qpd.proc_doorbells) {
-		err = kfd_alloc_process_doorbells(dev->kfd, pdd);
-		if (err) {
-			pr_debug("failed to allocate process doorbells\n");
-			goto err_bind_process;
-		}
+	err = kfd_alloc_process_doorbells(dev->kfd, pdd);
+	if (err) {
+		pr_debug("failed to allocate process doorbells\n");
+		goto err_bind_process;
 	}
 
 	err = kfd_queue_acquire_buffers(pdd, &q_properties);
@@ -1112,11 +1110,19 @@ static int kfd_ioctl_get_available_memory(struct file *filep,
 {
 	struct kfd_ioctl_get_available_memory_args *args = data;
 	struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, args->gpu_id);
+	struct amdgpu_device *adev;
+	size_t available;
 
 	if (!pdd)
 		return -EINVAL;
-	args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev,
-							pdd->dev->node_id);
+
+	adev = pdd->dev->adev;
+	available = amdgpu_amdkfd_get_available_memory(adev, pdd->dev->node_id);
+	if (adev->apu_prefer_gtt)
+		args->available = ALIGN_DOWN(available, PAGE_SIZE);
+	else
+		args->available = ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN);
+
 	kfd_unlock_pdd(pdd);
 	return 0;
 }
@@ -1305,6 +1311,9 @@ static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
 		goto err_unlock;
 	}
 
+	/* Queues a reset could not stop may still access the buffer */
+	kfd_process_wait_gpu_reset_complete(p);
+
 	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev,
 				(struct kgd_mem *)mem, pdd->drm_priv, &size);
 
@@ -2455,11 +2464,9 @@ static int criu_restore_devices(struct kfd_process *p,
 			goto exit;
 		}
 
-		if (!pdd->qpd.proc_doorbells) {
-			ret = kfd_alloc_process_doorbells(dev->kfd, pdd);
-			if (ret)
-				goto exit;
-		}
+		ret = kfd_alloc_process_doorbells(dev->kfd, pdd);
+		if (ret)
+			goto exit;
 	}
 
 	/*
@@ -3223,7 +3230,7 @@ static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, v
 
 		break;
 	case KFD_IOC_DBG_TRAP_DISABLE:
-		r = kfd_dbg_trap_disable(target);
+		r = kfd_dbg_trap_disable(target, true);
 		break;
 	case KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT:
 		r = kfd_dbg_send_exception_to_runtime(target,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
index 0dd1fd448059..71b34a57c2cc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
@@ -374,7 +374,9 @@ int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd, bool sq_trap_en)
 
 	return amdgpu_mes_set_shader_debugger(pdd->dev->adev,
 					pdd->proc_ctx_gpu_addr, spi_dbg_cntl,
-					pdd->watch_points, flags, sq_trap_en,
+					pdd->watch_points,
+					pdd->watch_addresses,
+					flags, sq_trap_en,
 					ffs(pdd->dev->xcc_mask) - 1);
 }
 
@@ -449,6 +451,7 @@ int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd,
 	pdd->watch_points[watch_id] = pdd->dev->kfd2kgd->clear_address_watch(
 							pdd->dev->adev,
 							watch_id);
+	pdd->watch_addresses[watch_id] = 0;
 	amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 
 	if (!pdd->dev->kfd->shared_resources.enable_mes)
@@ -485,6 +488,7 @@ int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd,
 	}
 
 	amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
+	pdd->watch_addresses[*watch_id] = watch_address;
 	for_each_inst(xcc_id, xcc_mask)
 		pdd->watch_points[*watch_id] = pdd->dev->kfd2kgd->set_address_watch(
 				pdd->dev->adev,
@@ -655,7 +659,8 @@ void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind
 	kfd_dbg_set_workaround(target, false);
 }
 
-static void kfd_dbg_clean_exception_status(struct kfd_process *target)
+static void kfd_dbg_clean_exception_status(struct kfd_process *target,
+					   bool drain_irqs)
 {
 	struct process_queue_manager *pqm;
 	struct process_queue_node *pqn;
@@ -664,7 +669,8 @@ static void kfd_dbg_clean_exception_status(struct kfd_process *target)
 	for (i = 0; i < target->n_pdds; i++) {
 		struct kfd_process_device *pdd = target->pdds[i];
 
-		kfd_process_drain_interrupts(pdd);
+		if (drain_irqs)
+			kfd_process_drain_interrupts(pdd);
 
 		pdd->exception_status = 0;
 	}
@@ -680,7 +686,7 @@ static void kfd_dbg_clean_exception_status(struct kfd_process *target)
 	target->exception_status = 0;
 }
 
-int kfd_dbg_trap_disable(struct kfd_process *target)
+int kfd_dbg_trap_disable(struct kfd_process *target, bool drain_irqs)
 {
 	if (!target->debug_trap_enabled)
 		return 0;
@@ -704,7 +710,7 @@ int kfd_dbg_trap_disable(struct kfd_process *target)
 	}
 
 	target->debug_trap_enabled = false;
-	kfd_dbg_clean_exception_status(target);
+	kfd_dbg_clean_exception_status(target, drain_irqs);
 	kfd_unref_process(target);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.h b/drivers/gpu/drm/amd/amdkfd/kfd_debug.h
index fbb751821c69..5babf9b992cd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.h
@@ -43,7 +43,7 @@ bool kfd_dbg_ev_raise(uint64_t event_mask,
 			unsigned int source_id, bool use_worker,
 			void *exception_data,
 			size_t exception_data_size);
-int kfd_dbg_trap_disable(struct kfd_process *target);
+int kfd_dbg_trap_disable(struct kfd_process *target, bool drain_irqs);
 int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd,
 			void __user *runtime_info,
 			uint32_t *runtime_info_size);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 5e57576fba21..ce67e56efa8c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -4215,6 +4215,15 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
 		     queue++) {
 			r = dqm->dev->kfd2kgd->hqd_sdma_dump(
 				dqm->dev->adev, pipe, queue, &dump, &n_regs);
+			if (r == -EINVAL) {
+				/*
+				 * SDMA engine not present/mapped on this
+				 * device; skip it instead of aborting the
+				 * whole dump.
+				 */
+				r = 0;
+				break;
+			}
 			if (r)
 				break;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index fdcf7f2d1b5b..6af268a031f2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -239,11 +239,9 @@ phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd)
 	struct amdgpu_device *adev = pdd->dev->adev;
 	uint32_t first_db_index;
 
-	if (!pdd->qpd.proc_doorbells) {
-		if (kfd_alloc_process_doorbells(pdd->dev->kfd, pdd))
-			/* phys_addr_t 0 is error */
-			return 0;
-	}
+	if (kfd_alloc_process_doorbells(pdd->dev->kfd, pdd))
+		/* phys_addr_t 0 is error */
+		return 0;
 
 	first_db_index = amdgpu_doorbell_index_on_bar(adev,
 						      pdd->qpd.proc_doorbells,
@@ -257,12 +255,21 @@ int kfd_alloc_process_doorbells(struct kfd_dev *kfd, struct kfd_process_device *
 	int r;
 	struct qcm_process_device *qpd = &pdd->qpd;
 
+	mutex_lock(&pdd->doorbell_mutex);
+
+	/* Another thread may have won the check-and-allocate race */
+	if (qpd->proc_doorbells) {
+		mutex_unlock(&pdd->doorbell_mutex);
+		return 0;
+	}
+
 	/* Allocate bitmap for dynamic doorbell allocation */
 	qpd->doorbell_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
 					     GFP_KERNEL);
 	if (!qpd->doorbell_bitmap) {
 		DRM_ERROR("Failed to allocate process doorbell bitmap\n");
-		return -ENOMEM;
+		r = -ENOMEM;
+		goto unlock;
 	}
 
 	r = init_doorbell_bitmap(&pdd->qpd, kfd);
@@ -284,11 +291,15 @@ int kfd_alloc_process_doorbells(struct kfd_dev *kfd, struct kfd_process_device *
 		DRM_ERROR("Failed to allocate process doorbells\n");
 		goto err;
 	}
+
+	mutex_unlock(&pdd->doorbell_mutex);
 	return 0;
 
 err:
 	bitmap_free(qpd->doorbell_bitmap);
 	qpd->doorbell_bitmap = NULL;
+unlock:
+	mutex_unlock(&pdd->doorbell_mutex);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
index 19406ab92c5b..45ac77bb2102 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v10.c
@@ -376,7 +376,7 @@ static void event_interrupt_wq_v10(struct kfd_node *dev,
 						&exception_data,
 						sizeof(exception_data));
 	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
-		kfd_process_close_interrupt_drain(pasid);
+		amdgpu_pasid_drain_irq_done(pasid);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
index 12d81abed748..24d13a74a181 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
@@ -408,7 +408,7 @@ static void event_interrupt_wq_v11(struct kfd_node *dev,
 		}
 
 	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
-		kfd_process_close_interrupt_drain(pasid);
+		amdgpu_pasid_drain_irq_done(pasid);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
index 0da7e1db55c9..4c7ad5aaaebc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
@@ -395,7 +395,7 @@ static void event_interrupt_wq_v12_1(struct kfd_node *node,
 		}
 
 	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
-		kfd_process_close_interrupt_drain(pasid);
+		amdgpu_pasid_drain_irq_done(pasid);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
index 2ae1129fea55..8dee8c43824d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
@@ -124,6 +124,28 @@ enum SQ_INTERRUPT_ERROR_TYPE {
 #define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20
 
 /*
+ * PTL (Peak TOPS Limiter) register skip notification.
+ * When FW skips a privileged register access, it sends interrupt 177
+ * with GENERIC1 and provides packet type info in CPC_INT_CNTX_ID[31:28].
+ */
+#define CPC_PTL_PACKET_TYPE_SET_UCONFIG_REG	1
+#define CPC_PTL_PACKET_TYPE_SET_SH_REG		2
+#define CPC_PTL_PACKET_TYPE_COPY_DATA_READ	3
+#define CPC_PTL_PACKET_TYPE_COPY_DATA_WRITE	4
+#define CPC_PTL_PACKET_TYPE_WRITE_DATA		5
+#define CPC_PTL_PACKET_TYPE_COND_WRITE		6
+#define CPC_PTL_PACKET_TYPE_WAIT_REG_MEM	7
+#define CPC_PTL_PACKET_TYPE_WAIT_REG_MEM64	8
+
+/* CPC_INT_CNTX_ID[31:28]: packet type that was skipped */
+#define CPC_INT_CNTX_ID__PACKET_TYPE__SHIFT	28
+#define CPC_INT_CNTX_ID__PACKET_TYPE_MASK	0xF0000000
+
+/* CPC_INT_INFO[30:28]: queue ID */
+#define CPC_INT_INFO__QUEUE_ID__SHIFT		28
+#define CPC_INT_INFO__QUEUE_ID_MASK		0x70000000
+
+/*
  * The debugger will send user data(m0) with PRIV=1 to indicate it requires
  * notification from the KFD with the following queue id (DOORBELL_ID) and
  * trap code (TRAP_CODE).
@@ -142,6 +164,30 @@ enum SQ_INTERRUPT_ERROR_TYPE {
 				KFD_DEBUG_CP_BAD_OP_ECODE_MASK)		\
 				>> KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT)
 
+static const char *cpc_packet_type_name(uint32_t packet_type)
+{
+	switch (packet_type) {
+	case CPC_PTL_PACKET_TYPE_SET_UCONFIG_REG:
+		return "SET_UCONFIG_REG";
+	case CPC_PTL_PACKET_TYPE_SET_SH_REG:
+		return "SET_SH_REG";
+	case CPC_PTL_PACKET_TYPE_COPY_DATA_READ:
+		return "COPY_DATA_READ";
+	case CPC_PTL_PACKET_TYPE_COPY_DATA_WRITE:
+		return "COPY_DATA_WRITE";
+	case CPC_PTL_PACKET_TYPE_WRITE_DATA:
+		return "WRITE_DATA";
+	case CPC_PTL_PACKET_TYPE_COND_WRITE:
+		return "COND_WRITE";
+	case CPC_PTL_PACKET_TYPE_WAIT_REG_MEM:
+		return "WAIT_REG_MEM";
+	case CPC_PTL_PACKET_TYPE_WAIT_REG_MEM64:
+		return "WAIT_REG_MEM64";
+	default:
+		return "UNKNOWN";
+	}
+}
+
 static void event_interrupt_poison_consumption_v9(struct kfd_node *dev,
 				uint16_t pasid, uint16_t client_id)
 {
@@ -352,6 +398,8 @@ static bool event_interrupt_isr_v9(struct kfd_node *dev,
 		source_id == SOC15_INTSRC_SDMA_ECC ||
 		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
 		source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
+		(client_id == SOC15_IH_CLIENTID_GRBM_CP &&
+		 source_id == SOC15_INTSRC_CP_PTL_REG_SKIP) ||
 		KFD_IRQ_IS_FENCE(client_id, source_id) ||
 		((client_id == SOC15_IH_CLIENTID_VMC ||
 		client_id == SOC15_IH_CLIENTID_VMC1 ||
@@ -538,6 +586,34 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
 				KFD_DEBUG_DOORBELL_ID(context_id0),
 				KFD_EC_MASK(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0)),
 				NULL, 0);
+		} else if (source_id == SOC15_INTSRC_CP_PTL_REG_SKIP) {
+			uint32_t packet_type, queue_id;
+			uint32_t skipped_addr;
+
+			/*
+			 * PTL (Peak TOPS Limiter) register skip notification.
+			 * FW skipped a privileged register access and is notifying KFD.
+			 * Data from IH entry (not registers):
+			 * - context_id0 contains CPC_INT_CNTX_ID with packet type in [31:28]
+			 * - context_id1 contains CPC_INT_INFO with queue ID in [30:28]
+			 * - context_id2 contains CPC_INT_ADDR (skipped register address)
+			 * - PASID comes from CPC_INT_PASID (already extracted from IH entry)
+			 */
+			skipped_addr = SOC15_CONTEXT_ID2_FROM_IH_ENTRY(ih_ring_entry);
+
+			/* Extract packet type from CPC_INT_CNTX_ID[31:28] */
+			packet_type = (context_id0 & CPC_INT_CNTX_ID__PACKET_TYPE_MASK) >>
+					CPC_INT_CNTX_ID__PACKET_TYPE__SHIFT;
+
+			/* Extract queue ID from CPC_INT_INFO[30:28] */
+			queue_id = (context_id1 & CPC_INT_INFO__QUEUE_ID_MASK) >>
+					CPC_INT_INFO__QUEUE_ID__SHIFT;
+
+			dev_info_ratelimited(dev->adev->dev,
+				"PTL: AQL request was rejected due to incompatible user input; please verify your PTL configuration.");
+			dev_dbg_ratelimited(dev->adev->dev,
+				"PTL: register access skipped - pasid 0x%x, queue %u, packet %s, addr 0x%x\n",
+				pasid, queue_id, cpc_packet_type_name(packet_type), skipped_addr);
 		}
 	} else if (client_id == SOC15_IH_CLIENTID_SDMA0 ||
 		   client_id == SOC15_IH_CLIENTID_SDMA1 ||
@@ -596,7 +672,7 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
 						sizeof(exception_data));
 		kfd_smi_event_update_vmfault(dev, pasid);
 	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
-		kfd_process_close_interrupt_drain(pasid);
+		amdgpu_pasid_drain_irq_done(pasid);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index d508ac5a18f8..2bbf5e63f615 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -42,9 +42,9 @@ static inline struct v12_1_compute_mqd *get_mqd(void *mqd)
 	return (struct v12_1_compute_mqd *)mqd;
 }
 
-static inline struct v12_sdma_mqd *get_sdma_mqd(void *mqd)
+static inline struct v12_1_sdma_mqd *get_sdma_mqd(void *mqd)
 {
-	return (struct v12_sdma_mqd *)mqd;
+	return (struct v12_1_sdma_mqd *)mqd;
 }
 
 static void mqd_symmetrically_map_cu_mask_v12_1(struct mqd_manager *mm,
@@ -408,9 +408,9 @@ static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
 		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
 		struct queue_properties *q)
 {
-	struct v12_sdma_mqd *m;
+	struct v12_1_sdma_mqd *m;
 
-	m = (struct v12_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+	m = (struct v12_1_sdma_mqd *) mqd_mem_obj->cpu_ptr;
 
 	memset(m, 0, PAGE_SIZE);
 
@@ -427,39 +427,41 @@ static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
 		struct queue_properties *q,
 		struct mqd_update_info *minfo)
 {
-	struct v12_sdma_mqd *m;
+	struct v12_1_sdma_mqd *m;
 
 	m = get_sdma_mqd(mqd);
-	m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
+	m->sdmax_queuen_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
 		<< SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
 		q->vmid << SDMA0_SDMA_QUEUE0_RB_CNTL__RB_VMID__SHIFT |
 		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
 		6 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
 		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT;
 
-	m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
-	m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
-	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
-	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
-	m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
-	m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
-	m->sdmax_rlcx_doorbell_offset =
+	m->sdmax_queuen_rb_base = lower_32_bits(q->queue_address >> 8);
+	m->sdmax_queuen_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+	m->sdmax_queuen_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_queuen_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_queuen_rb_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
+	m->sdmax_queuen_rb_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
+	m->sdmax_queuen_doorbell_offset =
 		q->doorbell_off << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
 
-	m->sdmax_rlcx_sched_cntl = (amdgpu_sdma_phase_quantum
+	m->sdmax_queuen_sched_cntl = (amdgpu_sdma_phase_quantum
 		<< SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT)
 		 & SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK;
 
 	m->sdma_engine_id = q->sdma_engine_id;
 	m->sdma_queue_id = q->sdma_queue_id;
 
-	m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
+	m->sdmax_queuen_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
+
+	m->sdmax_queuen_dummy_reg1 = 1;
 
 	/* Allow context switch so we don't cross-process starve with a massive
 	* command buffer of long-running SDMA commands
-	* sdmax_rlcx_ib_cntl represent SDMA_QUEUE0_IB_CNTL register
+	* sdmax_queuen_ib_cntl represent SDMA_QUEUE0_IB_CNTL register
 	*/
-	m->sdmax_rlcx_ib_cntl |= SDMA0_SDMA_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB_MASK;
+	m->sdmax_queuen_ib_cntl |= SDMA0_SDMA_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB_MASK;
 
 	q->is_active = QUEUE_IS_ACTIVE(*q);
 }
@@ -662,7 +664,7 @@ static int debugfs_show_mqd(struct seq_file *m, void *data)
 static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
 {
 	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
-		     data, sizeof(struct v12_sdma_mqd), false);
+		     data, sizeof(struct v12_1_sdma_mqd), false);
 	return 0;
 }
 
@@ -806,7 +808,7 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
 		mqd->update_mqd = update_mqd_sdma;
 		mqd->destroy_mqd = kfd_destroy_mqd_sdma;
 		mqd->is_occupied = kfd_is_occupied_sdma;
-		mqd->mqd_size = sizeof(struct v12_sdma_mqd);
+		mqd->mqd_size = sizeof(struct v12_1_sdma_mqd);
 		mqd->mqd_stride = kfd_mqd_stride;
 		mqd->restore_mqd = restore_mqd_sdma_v12_1;
 #if defined(CONFIG_DEBUG_FS)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index b9a31489b1c8..f0ce5fc4a138 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -784,6 +784,9 @@ struct kfd_process_device {
 	/* per-process-per device QCM data structure */
 	struct qcm_process_device qpd;
 
+	/* Serializes process doorbell allocation for this device */
+	struct mutex doorbell_mutex;
+
 	/*Apertures*/
 	uint64_t lds_base;
 	uint64_t lds_limit;
@@ -870,6 +873,7 @@ struct kfd_process_device {
 	uint32_t spi_dbg_override;
 	uint32_t spi_dbg_launch_mode;
 	uint32_t watch_points[4];
+	uint64_t watch_addresses[4];
 	uint32_t alloc_watch_ids;
 
 	/*
@@ -1037,10 +1041,6 @@ struct kfd_process {
 	uint64_t exception_enable_mask;
 	uint64_t exception_status;
 
-	/* Used to drain stale interrupts */
-	wait_queue_head_t wait_irq_drain;
-	bool irq_drain_is_open;
-
 	/* shared virtual memory registered by this process */
 	struct svm_range_list svms;
 
@@ -1130,6 +1130,7 @@ void kfd_unref_process(struct kfd_process *p);
 int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger);
 int kfd_process_restore_queues(struct kfd_process *p);
 void kfd_suspend_all_processes(void);
+void kfd_process_wait_gpu_reset_complete(struct kfd_process *p);
 int kfd_resume_all_processes(void);
 
 struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *process,
@@ -1252,9 +1253,7 @@ bool enqueue_ih_ring_entry(struct kfd_node *kfd, const void *ih_ring_entry);
 bool interrupt_is_wanted(struct kfd_node *dev,
 				const uint32_t *ih_ring_entry,
 				uint32_t *patched_ihre, bool *flag);
-int kfd_process_drain_interrupts(struct kfd_process_device *pdd);
-void kfd_process_close_interrupt_drain(unsigned int pasid);
-
+void kfd_process_drain_interrupts(struct kfd_process_device *pdd);
 /* amdkfd Apertures */
 int kfd_init_apertures(struct kfd_process *process);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 226f52626e84..ac18c2be6ecb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1019,8 +1019,6 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
 		if (ret)
 			pr_warn("Failed to create debugfs entry for the kfd_process, ret = %d\n",
 				ret);
-
-		init_waitqueue_head(&process->wait_irq_drain);
 	}
 out:
 	mutex_unlock(&kfd_processes_mutex);
@@ -1233,6 +1231,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
 		}
 
 		atomic_dec(&pdd->dev->kfd->kfd_processes_count);
+		mutex_destroy(&pdd->doorbell_mutex);
 
 		kfree(pdd);
 		p->pdds[i] = NULL;
@@ -1293,7 +1292,7 @@ static void kfd_process_remove_sysfs(struct kfd_process *p)
 /*
  * If any GPU is ongoing reset, wait for reset complete.
  */
-static void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
+void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
 {
 	int i;
 
@@ -1420,7 +1419,7 @@ void kfd_process_notifier_release_internal(struct kfd_process *p)
 	 * back on the MES list and, once proc_ctx_bo is freed, MES would fault
 	 * on the freed process context. Retiring the context last avoids this.
 	 */
-	kfd_dbg_trap_disable(p);
+	kfd_dbg_trap_disable(p, false);
 
 	if (atomic_read(&p->debugged_process_count) > 0) {
 		struct kfd_process *target;
@@ -1430,7 +1429,7 @@ void kfd_process_notifier_release_internal(struct kfd_process *p)
 		hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
 			if (target->debugger_process && target->debugger_process == p) {
 				mutex_lock_nested(&target->mutex, 1);
-				kfd_dbg_trap_disable(target);
+				kfd_dbg_trap_disable(target, true);
 				mutex_unlock(&target->mutex);
 				if (atomic_read(&p->debugged_process_count) == 0)
 					break;
@@ -1802,6 +1801,7 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
 	pdd->dev = dev;
 	INIT_LIST_HEAD(&pdd->qpd.queues_list);
 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
+	mutex_init(&pdd->doorbell_mutex);
 	pdd->qpd.dqm = dev->dqm;
 	pdd->qpd.pqm = &p->pqm;
 	pdd->qpd.evicted = 0;
@@ -2313,17 +2313,14 @@ int kfd_resume_all_processes(void)
 	return ret;
 }
 
-/* assumes caller holds process lock. */
-int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
+void kfd_process_drain_interrupts(struct kfd_process_device *pdd)
 {
 	uint32_t irq_drain_fence[8];
 	uint8_t node_id = 0;
-	int r = 0;
+	int r;
 
-	if (!KFD_IS_SOC15(pdd->dev))
-		return 0;
-
-	pdd->process->irq_drain_is_open = true;
+	if (!KFD_IS_SOC15(pdd->dev) || !pdd->drm_priv)
+		return;
 
 	memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
 	irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
@@ -2342,32 +2339,12 @@ int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
 	}
 
 	/* ensure stale irqs scheduled KFD interrupts and send drain fence. */
-	if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
-						     irq_drain_fence)) {
-		pdd->process->irq_drain_is_open = false;
-		return 0;
-	}
-
-	r = wait_event_interruptible(pdd->process->wait_irq_drain,
-				     !READ_ONCE(pdd->process->irq_drain_is_open));
+	r = amdgpu_pasid_drain_irq(pdd->dev->adev, pdd->pasid, irq_drain_fence,
+				   AMDGPU_FENCE_JIFFIES_TIMEOUT);
 	if (r)
-		pdd->process->irq_drain_is_open = false;
-
-	return r;
-}
-
-void kfd_process_close_interrupt_drain(unsigned int pasid)
-{
-	struct kfd_process *p;
-
-	p = kfd_lookup_process_by_pasid(pasid, NULL);
-
-	if (!p)
-		return;
-
-	WRITE_ONCE(p->irq_drain_is_open, false);
-	wake_up_all(&p->wait_irq_drain);
-	kfd_unref_process(p);
+		dev_warn_ratelimited(pdd->dev->adev->dev,
+				     "irq drain on pasid 0x%x did not complete (%d)\n",
+				     pdd->pasid, r);
 }
 
 struct send_exception_work_handler_workarea {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index ae3e7c54014a..6010a5d7a7eb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -230,6 +230,15 @@ void kfd_queue_buffer_put(struct amdgpu_bo **bo)
 	amdgpu_bo_unref(bo);
 }
 
+static void kfd_queue_put_bos(struct queue_properties *properties)
+{
+	kfd_queue_buffer_put(&properties->wptr_bo);
+	kfd_queue_buffer_put(&properties->rptr_bo);
+	kfd_queue_buffer_put(&properties->ring_bo);
+	kfd_queue_buffer_put(&properties->eop_buf_bo);
+	kfd_queue_buffer_put(&properties->cwsr_bo);
+}
+
 int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_properties *properties)
 {
 	struct kfd_topology_device *topo_dev;
@@ -343,7 +352,13 @@ out_err_release:
 	 * dropping the VM reservation.
 	 */
 	kfd_queue_unref_bo_vas(pdd, properties);
-	kfd_queue_release_buffers(pdd, properties);
+	/*
+	 * The SVM CWSR range is the last thing acquired and a failed
+	 * kfd_queue_buffer_svm_get() takes nothing, so only the BOs are held
+	 * here. Dropping the SVM reference would drop the one of another queue
+	 * using the same CWSR address.
+	 */
+	kfd_queue_put_bos(properties);
 	return err;
 }
 
@@ -352,11 +367,7 @@ int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_prope
 	struct kfd_topology_device *topo_dev;
 	u64 total_cwsr_size;
 
-	kfd_queue_buffer_put(&properties->wptr_bo);
-	kfd_queue_buffer_put(&properties->rptr_bo);
-	kfd_queue_buffer_put(&properties->ring_bo);
-	kfd_queue_buffer_put(&properties->eop_buf_bo);
-	kfd_queue_buffer_put(&properties->cwsr_bo);
+	kfd_queue_put_bos(properties);
 
 	topo_dev = kfd_topology_device_by_id(pdd->dev->id);
 	if (!topo_dev)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index cfb428c72686..1f5d73f0ebce 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -2158,14 +2158,25 @@ static struct svm_range *svm_range_clone(struct svm_range *old)
 		svm_range_free(new, false);
 		return NULL;
 	}
-	if (old->svm_bo) {
+
+	/* old->lock keeps old->svm_bo alive while taking the reference */
+	mutex_lock(&old->lock);
+	if (svm_bo_ref_unless_zero(old->svm_bo)) {
 		new->ttm_res = old->ttm_res;
 		new->offset = old->offset;
-		new->svm_bo = svm_range_bo_ref(old->svm_bo);
+		new->svm_bo = old->svm_bo;
 		spin_lock(&new->svm_bo->list_lock);
 		list_add(&new->svm_bo_list, &new->svm_bo->range_list);
 		spin_unlock(&new->svm_bo->list_lock);
 	}
+	mutex_unlock(&old->lock);
+
+	/* prange only holds an svm_bo reference while ttm_res is set, unref
+	 * outside old->lock, svm_range_bo_release takes it and would deadlock.
+	 */
+	if (new->svm_bo && !new->ttm_res)
+		svm_range_bo_unref(new->svm_bo);
+
 	new->flags = old->flags;
 	new->preferred_loc = old->preferred_loc;
 	new->prefetch_loc = old->prefetch_loc;
diff --git a/drivers/gpu/drm/amd/amdkfd/soc15_int.h b/drivers/gpu/drm/amd/amdkfd/soc15_int.h
index e5c0205f2618..736b02ccdefb 100644
--- a/drivers/gpu/drm/amd/amdkfd/soc15_int.h
+++ b/drivers/gpu/drm/amd/amdkfd/soc15_int.h
@@ -25,6 +25,7 @@
 
 #include "soc15_ih_clientid.h"
 
+#define SOC15_INTSRC_CP_PTL_REG_SKIP	177
 #define SOC15_INTSRC_CP_END_OF_PIPE	181
 #define SOC15_INTSRC_CP_BAD_OPCODE	183
 #define SOC15_INTSRC_SQ_INTERRUPT_MSG	239
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 5b634ae07c0d..d740108b1274 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -2614,13 +2614,6 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 
 				if (psr_feature_enabled) {
 					amdgpu_dm_set_psr_caps(link, aconnector);
-					drm_info(adev_to_drm(adev), "%s: PSR support %d, DC PSR ver %d, sink PSR ver %d DPCD caps 0x%x su_y_granularity %d\n",
-						 aconnector->base.name,
-						 link->psr_settings.psr_feature_enabled,
-						 link->psr_settings.psr_version,
-						 link->dpcd_caps.psr_info.psr_version,
-						 link->dpcd_caps.psr_info.psr_dpcd_caps.raw,
-						 link->dpcd_caps.psr_info.psr2_su_y_granularity_cap);
 				}
 			}
 		}
@@ -4460,12 +4453,12 @@ STATIC_IFN_KUNIT void dm_clear_writeback(struct amdgpu_display_manager *dm,
 EXPORT_IF_KUNIT(dm_clear_writeback);
 
 /**
- * amdgpu_dm_mod_power_update_streams - update mod_power stream state on modeset
+ * amdgpu_dm_mod_power_update_streams - update mod_power stream state
  * @state: the drm atomic state
  * @dm: the display manager to update mod_power on
  *
- * Notify mod_power of stream changes on modeset events, and disable PSR/Replay
- * in preparation for hardware programming. See also
+ * Disable PSR/Replay before hardware programming for active streams, and
+ * update mod_power stream state when a modeset changes the stream. See also
  * amdgpu_dm_mod_power_setup_streams() for post-modeset mod_power setup.
  */
 STATIC_IFN_KUNIT void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commit *state,
@@ -4481,15 +4474,15 @@ STATIC_IFN_KUNIT void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commi
 		dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
 		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
 
-		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
-			continue;
-
 		/*
-		 * Update mod_power on modeset event in preparation for hw
-		 * programming. Always use the old stream, since it would have
-		 * been previously added to mod_power. If old stream is null (on
-		 * crtc enable, for example), mod_power will no-op, which is the
-		 * desried behavior.
+		 * Notify mod_power that hardware programming is about to occur.
+		 * This must be done before the modeset check because a non-modeset
+		 * commit can still update an active framebuffer or plane while
+		 * PSR/Replay is enabled.
+		 *
+		 * Always use the old stream since it is the stream currently
+		 * registered with mod_power. If old stream is NULL, the event
+		 * helpers will no-op.
 		 */
 		if (old_crtc_state->active) {
 			scoped_guard(mutex, &dm->dc_lock) {
@@ -4498,11 +4491,15 @@ STATIC_IFN_KUNIT void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commi
 					psr_event_hw_programming, true);
 				amdgpu_dm_replay_set_event(dm, dm_old_crtc_state->stream, true,
 					replay_event_hw_programming, true);
-				amdgpu_dm_replay_set_event(dm, dm_old_crtc_state->stream, false,
-					replay_event_general_ui, false);
 			}
 		}
 
+		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+		if (dm_old_crtc_state->stream == dm_new_crtc_state->stream)
+			continue;
+
 		if (new_crtc_state->active) {
 			aconnector = (struct amdgpu_dm_connector *)
 				dm_new_crtc_state->stream->dm_stream_context;
@@ -4549,12 +4546,36 @@ STATIC_IFN_KUNIT void amdgpu_dm_mod_power_setup_streams(struct drm_atomic_commit
 			continue;
 
 		if (new_crtc_state->active) {
+			struct amdgpu_dm_connector *aconnector =
+				(struct amdgpu_dm_connector *)
+				dm_new_crtc_state->stream->dm_stream_context;
+			struct dc_link *link = dm_new_crtc_state->stream->link;
+
 			amdgpu_dm_link_setup_replay(dm_new_crtc_state->stream,
 					&acrtc->dm_irq_params.vrr_params);
 			mod_power_notify_mode_change(dm->power_module,
 						dm_new_crtc_state->stream,
 						false);
 
+			if (link->replay_settings.config.replay_cap_support) {
+				drm_info(state->dev,
+					"%s: Replay enabled %d, Allow active %d, Config ver %d\n",
+					aconnector->base.name,
+					link->replay_settings.replay_feature_enabled,
+					link->replay_settings.replay_allow_active,
+					link->replay_settings.config.replay_version);
+			}
+			if (link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED) {
+				drm_info(state->dev,
+					"%s: PSR support %d, DC PSR ver %d, Sink PSR ver %d, DPCD caps 0x%x, su_y_granularity %d\n",
+					aconnector->base.name,
+					link->psr_settings.psr_feature_enabled,
+					link->psr_settings.psr_version,
+					link->dpcd_caps.psr_info.psr_version,
+					link->dpcd_caps.psr_info.psr_dpcd_caps.raw,
+					link->dpcd_caps.psr_info.psr2_su_y_granularity_cap);
+			}
+
 			/*
 			 * Block PSR / Replay on the new stream until display settles post-modeset.
 			 * These events will be cleared by amdgpu_dm_enable_self_refresh() once
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
index 2b81396bd06e..c78e26a145e3 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
@@ -1395,6 +1395,21 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
 }
 #endif
 
+STATIC_IFN_KUNIT bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector,
+					const struct drm_display_mode *mode)
+{
+	struct drm_display_mode *m;
+
+	list_for_each_entry(m, &aconnector->base.probed_modes, head) {
+		if (drm_mode_equal(m, mode))
+			return true;
+	}
+
+	return false;
+}
+
+EXPORT_IF_KUNIT(is_duplicate_mode);
+
 void amdgpu_dm_update_stream_scaling_settings(struct drm_device *dev,
 					   const struct drm_display_mode *mode,
 					   const struct dm_connector_state *dm_state,
@@ -1464,13 +1479,14 @@ create_stream_for_sink(struct drm_connector *connector,
 	struct drm_device *dev = connector->dev;
 	struct amdgpu_dm_connector *aconnector = NULL;
 	struct drm_display_mode *preferred_mode = NULL;
+	struct drm_display_mode *connector_mode;
 	const struct drm_connector_state *con_state = &dm_state->base;
 	struct dc_stream_state *stream = NULL;
 	struct drm_display_mode mode;
 	struct drm_display_mode saved_mode;
 	struct drm_display_mode *freesync_mode = NULL;
-	bool native_mode_found = false;
 	bool recalculate_timing = false;
+	bool unadvertised_mode = false;
 	bool scale = dm_state->scaling != RMX_OFF;
 	int mode_refresh;
 	int preferred_refresh = 0;
@@ -1520,14 +1536,14 @@ create_stream_for_sink(struct drm_connector *connector,
 	stream->timing.flags.LTE_340MCSC_SCRAMBLE =
 		connector->display_info.hdmi.scdc.scrambling.low_rates;
 
-	list_for_each_entry(preferred_mode, &connector->modes, head) {
+	list_for_each_entry(connector_mode, &connector->modes, head) {
 		/* Search for preferred mode */
-		if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED) {
-			native_mode_found = true;
+		if (connector_mode->type & DRM_MODE_TYPE_PREFERRED) {
+			preferred_mode = connector_mode;
 			break;
 		}
 	}
-	if (!native_mode_found)
+	if (!preferred_mode)
 		preferred_mode = list_first_entry_or_null(
 				&connector->modes,
 				struct drm_display_mode,
@@ -1557,6 +1573,13 @@ create_stream_for_sink(struct drm_connector *connector,
 					&mode, preferred_mode, scale);
 
 			preferred_refresh = drm_mode_vrefresh(preferred_mode);
+			unadvertised_mode = !is_duplicate_mode(aconnector, drm_mode);
+
+			if (unadvertised_mode && dm_state->scaling == RMX_OFF)
+				drm_dbg_driver(dev,
+					       "Unadvertised mode %dx%d@%d with RMX_OFF\n",
+					       drm_mode->hdisplay, drm_mode->vdisplay,
+					       drm_mode_vrefresh(drm_mode));
 		}
 	}
 
@@ -3005,19 +3028,6 @@ STATIC_IFN_KUNIT void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *co
 }
 EXPORT_IF_KUNIT(amdgpu_dm_connector_ddc_get_modes);
 
-STATIC_IFN_KUNIT bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector,
-			      struct drm_display_mode *mode)
-{
-	struct drm_display_mode *m;
-
-	list_for_each_entry(m, &aconnector->base.probed_modes, head) {
-		if (drm_mode_equal(m, mode))
-			return true;
-	}
-
-	return false;
-}
-EXPORT_IF_KUNIT(is_duplicate_mode);
 
 STATIC_IFN_KUNIT uint add_fs_modes(struct amdgpu_dm_connector *aconnector)
 {
@@ -3906,12 +3916,6 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
 			amdgpu_dm_connector->min_vfreq;
 	}
 
-	/* HDMI/PCON without an EDID FreeSync VCP code is not FreeSync-capable. */
-	if ((sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A ||
-	     as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) &&
-	    !sink->edid_caps.freesync_vcp_code)
-		freesync_capable = false;
-
 	/* Handle MCCS */
 	if (do_mccs) {
 		dm_helpers_read_mccs_caps(adev->dm.dc->ctx, amdgpu_dm_connector->dc_link, sink);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
index 79698deacf61..7bfbb6853833 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
@@ -214,7 +214,7 @@ bool adjust_colour_depth_from_display_info(struct dc_crtc_timing *timing_out,
 					   const struct drm_display_info *info);
 
 int to_drm_connector_type(enum signal_type st, uint32_t connector_id);
-bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector, struct drm_display_mode *mode);
+bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector, const struct drm_display_mode *mode);
 enum dc_aspect_ratio get_aspect_ratio(const struct drm_display_mode *mode_in);
 void copy_crtc_timing_for_drm_display_mode(const struct drm_display_mode *src_mode,
 					   struct drm_display_mode *dst_mode);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 36f32ed4b87c..6622a686e5e0 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -2995,64 +2995,6 @@ static ssize_t hdmi_cec_state_write(struct file *f, const char __user *buf,
 	return size;
 }
 
-/**
- * hdmi_automation_enable - Enable/Disable HDMI automation feature
- * @f: file structure.
- * @buf: userspace buffer. set to '1' to enable; '0' to disable automation feature.
- * @size: size of buffer from userpsace.
- * @pos: unused.
- *
- * Return size on success, error code on failure
- */
-static ssize_t hdmi_automation_enable(struct file *f, const char __user *buf,
-	size_t size, loff_t *pos)
-{
-	struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private;
-	char *wr_buf = NULL;
-	const uint32_t wr_buf_size = 40;
-	int max_param_num = 1;
-	uint8_t param_nums = 0;
-	long param[2];
-	bool hdmi_comp_auto;
-
-	if (size == 0)
-		return -EINVAL;
-
-	wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL);
-	if (!wr_buf)
-		return -ENOSPC;
-
-	if (parse_write_buffer_into_params(wr_buf, wr_buf_size,
-					   (long *)param, buf,
-					   max_param_num,
-					   &param_nums)) {
-		kfree(wr_buf);
-		return -EINVAL;
-	}
-
-	if (param_nums <= 0) {
-		kfree(wr_buf);
-		DRM_DEBUG_DRIVER("user data not be read\n");
-		return -EINVAL;
-	}
-
-	switch (param[0]) {
-	case 0:
-		hdmi_comp_auto = false;
-		break;
-	case 1:
-	default:
-		hdmi_comp_auto = true;
-		break;
-	}
-
-	/* Persist setting across sink re-detection/hotplug. */
-	aconnector->hdmi_comp_auto = hdmi_comp_auto;
-
-	kfree(wr_buf);
-	return size;
-}
-
 DEFINE_SHOW_ATTRIBUTE(dp_dsc_fec_support);
 DEFINE_SHOW_ATTRIBUTE(dmub_fw_state);
 DEFINE_SHOW_ATTRIBUTE(dmub_tracebuffer);
@@ -3171,12 +3113,6 @@ static const struct file_operations dp_mst_link_settings_debugfs_fops = {
 	.llseek = default_llseek
 };
 
-static const struct file_operations hdmi_automation_debugfs_fops = {
-	.owner = THIS_MODULE,
-	.write = hdmi_automation_enable,
-	.llseek = default_llseek
-};
-
 static const struct {
 	char *name;
 	const struct file_operations *fops;
@@ -3209,8 +3145,7 @@ static const struct {
 	const struct file_operations *fops;
 } hdmi_debugfs_entries[] = {
 		{"hdcp_sink_capability", &hdcp_sink_capability_fops},
-		{"hdmi_cec_state", &hdmi_cec_state_fops},
-		{"hdmi_automation", &hdmi_automation_debugfs_fops}
+		{"hdmi_cec_state", &hdmi_cec_state_fops}
 };
 
 /*
@@ -3975,6 +3910,8 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector)
 					    0644, dir, connector,
 					    hdmi_debugfs_entries[i].fops);
 		}
+
+		debugfs_create_bool("hdmi_automation", 0644, dir, &connector->hdmi_comp_auto);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c
index ab9cfbce8d73..386ba88c57c6 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c
@@ -42,7 +42,7 @@
 bool amdgpu_dm_is_dc_timing_adjust_needed(struct dm_crtc_state *old_state,
 					  struct dm_crtc_state *new_state)
 {
-	if (new_state->stream->adjust.timing_adjust_pending)
+	if (new_state->stream->timing_adjust_pending)
 		return true;
 	if (new_state->freesync_config.state ==  VRR_STATE_ACTIVE_FIXED)
 		return true;
@@ -112,6 +112,30 @@ void amdgpu_dm_reset_freesync_config_for_crtc(
 }
 EXPORT_IF_KUNIT(amdgpu_dm_reset_freesync_config_for_crtc);
 
+/* FreeSync module gates ACTIVE_VARIABLE on this minimum refresh span (Hz). */
+#define AMDGPU_DM_VRR_MIN_REFRESH_RANGE 10
+
+static bool amdgpu_dm_is_vrr_timing_valid(const struct dc_stream_state *stream,
+					  int min_vfreq, int max_vfreq)
+{
+	u64 nominal_vfreq_uhz, usable_max_vfreq_uhz;
+
+	/* Without complete timing/range data, leave the existing decision. */
+	if (!stream || !stream->timing.pix_clk_100hz ||
+	    !stream->timing.h_total || !stream->timing.v_total ||
+	    min_vfreq <= 0 || max_vfreq <= 0)
+		return true;
+
+	/* Match the FreeSync module's nominal-rate calculation. */
+	nominal_vfreq_uhz = mod_freesync_calc_nominal_field_rate(stream);
+	usable_max_vfreq_uhz = min_t(u64, (u64)max_vfreq * 1000000,
+				     nominal_vfreq_uhz);
+
+	return usable_max_vfreq_uhz >=
+	       (u64)min_vfreq * 1000000 +
+	       (u64)AMDGPU_DM_VRR_MIN_REFRESH_RANGE * 1000000;
+}
+
 void amdgpu_dm_get_freesync_config_for_crtc(
 	struct dm_crtc_state *new_crtc_state,
 	struct dm_connector_state *new_con_state)
@@ -121,15 +145,30 @@ void amdgpu_dm_get_freesync_config_for_crtc(
 	struct drm_display_mode *mode = &new_crtc_state->base.mode;
 	int vrefresh = drm_mode_vrefresh(mode);
 	bool fs_vid_mode = false;
+	bool vrr_timing_valid;
 
 	if (new_con_state->base.connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
 		return;
 
 	aconnector = to_amdgpu_dm_connector(new_con_state->base.connector);
 
+	vrr_timing_valid = amdgpu_dm_is_vrr_timing_valid(new_crtc_state->stream,
+							 aconnector->min_vfreq,
+							 aconnector->max_vfreq);
+	if (!vrr_timing_valid)
+		drm_dbg_kms(new_con_state->base.connector->dev,
+			    "VRR invalid %s %s %dx%d@%d %d-%d clk=%u h=%u v=%u\n",
+			    new_con_state->base.connector->name, mode->name,
+			    mode->hdisplay, mode->vdisplay, vrefresh,
+			    aconnector->min_vfreq, aconnector->max_vfreq,
+			    new_crtc_state->stream->timing.pix_clk_100hz,
+			    new_crtc_state->stream->timing.h_total,
+			    new_crtc_state->stream->timing.v_total);
+
 	new_crtc_state->vrr_supported = new_con_state->freesync_capable &&
 					vrefresh >= aconnector->min_vfreq &&
-					vrefresh <= aconnector->max_vfreq;
+					vrefresh <= aconnector->max_vfreq &&
+					vrr_timing_valid;
 
 	if (new_crtc_state->vrr_supported) {
 		new_crtc_state->stream->ignore_msa_timing_param = true;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index 3217aa82bea2..9774ea33272d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -309,7 +309,7 @@ fill_dc_mst_payload_table_from_drm(struct dc_link *link,
 				break;
 			}
 		}
-		ASSERT(i != copy_of_link_table.stream_count);
+		/* Payload absent from HW table when never allocated: nothing to remove. */
 	}
 
 	/* Fill payload info*/
@@ -1423,7 +1423,7 @@ void dm_helpers_init_panel_settings(
 	panel_config->pps.extra_post_OUI_ms = 0;
 	// Feature DSC
 	panel_config->dsc.disable_dsc_edp = false;
-	panel_config->dsc.force_dsc_edp_policy = 0;
+	panel_config->dsc.force_dsc_edp_policy = 1;
 }
 EXPORT_IF_KUNIT(dm_helpers_init_panel_settings);
 
@@ -1437,6 +1437,9 @@ void dm_helpers_override_panel_settings(
 	if (amdgpu_dc_debug_mask & DC_DISABLE_DSC)
 		link->panel_config.dsc.disable_dsc_edp = true;
 
+	if (!dc_link_should_enable_fec(link))
+		link->panel_config.dsc.force_dsc_edp_policy = 0;
+
 	if (dc_get_edp_link_panel_inst(ctx->dc, link, &panel_inst) && panel_inst == 1) {
 		link->panel_config.psr.disable_psr = true;
 		link->panel_config.psr.disallow_psrsu = true;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_freesync_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_freesync_test.c
index 5ad50bc31613..616a2b385140 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_freesync_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_freesync_test.c
@@ -173,7 +173,7 @@ static void dm_test_dc_timing_adjust_pending(struct kunit *test)
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
 
 	new_state->stream = stream;
-	stream->adjust.timing_adjust_pending = 1;
+	stream->timing_adjust_pending = 1;
 
 	KUNIT_EXPECT_TRUE(test, amdgpu_dm_is_dc_timing_adjust_needed(old_state, new_state));
 }
@@ -341,6 +341,48 @@ static void dm_test_freesync_config_out_of_range(struct kunit *test)
 }
 
 /**
+ * dm_test_freesync_config_invalid_timing - Reject an invalid VRR timing
+ * @test: The KUnit test context
+ */
+static void dm_test_freesync_config_invalid_timing(struct kunit *test)
+{
+	struct dm_test_freesync_ctx *ctx = dm_test_freesync_ctx_alloc(test);
+
+	ctx->conn_state->freesync_capable = true;
+	ctx->aconnector->min_vfreq = 45;
+	ctx->aconnector->max_vfreq = 60;
+	ctx->stream->timing.pix_clk_100hz = 270000;
+	ctx->stream->timing.h_total = 864;
+	ctx->stream->timing.v_total = 625;
+	ctx->crtc_state->base.mode.clock = 27000;
+	ctx->crtc_state->base.mode.hdisplay = 720;
+	ctx->crtc_state->base.mode.vdisplay = 576;
+	ctx->crtc_state->base.mode.htotal = 864;
+	ctx->crtc_state->base.mode.vtotal = 625;
+
+	amdgpu_dm_get_freesync_config_for_crtc(ctx->crtc_state, ctx->conn_state);
+
+	KUNIT_EXPECT_FALSE(test, ctx->crtc_state->vrr_supported);
+	KUNIT_EXPECT_EQ(test, (int)ctx->crtc_state->freesync_config.state,
+			(int)VRR_STATE_UNSUPPORTED);
+
+	ctx->stream->timing.pix_clk_100hz = 5940000;
+	ctx->stream->timing.h_total = 4400;
+	ctx->stream->timing.v_total = 2250;
+	ctx->crtc_state->base.mode.clock = 594000;
+	ctx->crtc_state->base.mode.hdisplay = 3840;
+	ctx->crtc_state->base.mode.vdisplay = 2160;
+	ctx->crtc_state->base.mode.htotal = 4400;
+	ctx->crtc_state->base.mode.vtotal = 2250;
+
+	amdgpu_dm_get_freesync_config_for_crtc(ctx->crtc_state, ctx->conn_state);
+
+	KUNIT_EXPECT_TRUE(test, ctx->crtc_state->vrr_supported);
+	KUNIT_EXPECT_EQ(test, (int)ctx->crtc_state->freesync_config.state,
+			(int)VRR_STATE_INACTIVE);
+}
+
+/**
  * dm_test_freesync_config_active_variable - Test vrr_enabled yields ACTIVE_VARIABLE
  * @test: The KUnit test context
  */
@@ -350,7 +392,7 @@ static void dm_test_freesync_config_active_variable(struct kunit *test)
 
 	ctx->conn_state->freesync_capable = true;
 	ctx->aconnector->min_vfreq = 48;
-	ctx->aconnector->max_vfreq = 120;
+	ctx->aconnector->max_vfreq = 60;
 	ctx->crtc_state->base.vrr_enabled = true;
 
 	amdgpu_dm_get_freesync_config_for_crtc(ctx->crtc_state, ctx->conn_state);
@@ -362,7 +404,7 @@ static void dm_test_freesync_config_active_variable(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, ctx->crtc_state->freesync_config.min_refresh_in_uhz,
 			48000000U);
 	KUNIT_EXPECT_EQ(test, ctx->crtc_state->freesync_config.max_refresh_in_uhz,
-			120000000U);
+			60000000U);
 	KUNIT_EXPECT_TRUE(test, ctx->crtc_state->freesync_config.vsif_supported);
 	KUNIT_EXPECT_TRUE(test, ctx->crtc_state->freesync_config.btr);
 }
@@ -377,7 +419,7 @@ static void dm_test_freesync_config_inactive(struct kunit *test)
 
 	ctx->conn_state->freesync_capable = true;
 	ctx->aconnector->min_vfreq = 48;
-	ctx->aconnector->max_vfreq = 120;
+	ctx->aconnector->max_vfreq = 60;
 	ctx->crtc_state->base.vrr_enabled = false;
 
 	amdgpu_dm_get_freesync_config_for_crtc(ctx->crtc_state, ctx->conn_state);
@@ -397,7 +439,7 @@ static void dm_test_freesync_config_active_fixed(struct kunit *test)
 
 	ctx->conn_state->freesync_capable = true;
 	ctx->aconnector->min_vfreq = 48;
-	ctx->aconnector->max_vfreq = 120;
+	ctx->aconnector->max_vfreq = 60;
 	/* Pre-set fixed state selects the freesync-video (fixed) path */
 	ctx->crtc_state->freesync_config.state = VRR_STATE_ACTIVE_FIXED;
 	ctx->crtc_state->freesync_config.fixed_refresh_in_uhz = 60000000;
@@ -556,7 +598,7 @@ static void dm_test_update_freesync_state_pre_ai_active(struct kunit *test)
 
 	amdgpu_dm_update_freesync_state_on_stream(&adev->dm, &crtc_state, stream, surface, 0);
 
-	KUNIT_EXPECT_FALSE(test, stream->adjust.timing_adjust_pending);
+	KUNIT_EXPECT_FALSE(test, stream->timing_adjust_pending);
 }
 
 static void dm_test_update_freesync_state_pcon_version(struct kunit *test, u8 version)
@@ -842,6 +884,7 @@ static struct kunit_case amdgpu_dm_freesync_tests[] = {
 	KUNIT_CASE(dm_test_freesync_config_writeback),
 	KUNIT_CASE(dm_test_freesync_config_not_capable),
 	KUNIT_CASE(dm_test_freesync_config_out_of_range),
+	KUNIT_CASE(dm_test_freesync_config_invalid_timing),
 	KUNIT_CASE(dm_test_freesync_config_active_variable),
 	KUNIT_CASE(dm_test_freesync_config_inactive),
 	KUNIT_CASE(dm_test_freesync_config_active_fixed),
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
index d37fe26eecea..38f53c596f0d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
@@ -2092,7 +2092,12 @@ static void dm_test_init_panel_settings_dsc(struct kunit *test)
 	dm_helpers_init_panel_settings(NULL, &panel_config, sink);
 
 	KUNIT_EXPECT_FALSE(test, panel_config.dsc.disable_dsc_edp);
-	KUNIT_EXPECT_EQ(test, panel_config.dsc.force_dsc_edp_policy, 0U);
+	KUNIT_EXPECT_EQ(test, panel_config.dsc.force_dsc_edp_policy, 1U);
+}
+
+static bool dm_test_dp_should_enable_fec(const struct dc_link *link)
+{
+	return true;
 }
 
 /* Tests for dm_helpers_override_panel_settings() */
@@ -2106,14 +2111,20 @@ static void dm_test_override_panel_settings_debug_mask_disables_dsc(struct kunit
 	struct dc_context *ctx;
 	struct dc_link *link;
 	struct dc *dc;
+	struct link_service *link_srv;
 	uint old_debug_mask;
 
 	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, ctx);
 	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, dc);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
 	link = dm_kunit_alloc_link(test);
 	ctx->dc = dc;
+	dc->link_srv = link_srv;
+	link->dc = dc;
+	link_srv->dp_should_enable_fec = dm_test_dp_should_enable_fec;
 	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
 
 	old_debug_mask = dm_helpers_get_dc_debug_mask();
@@ -2134,15 +2145,22 @@ static void dm_test_override_panel_settings_second_edp_disables_psr(struct kunit
 	struct dc_link *first_link;
 	struct dc_link *second_link;
 	struct dc *dc;
+	struct link_service *link_srv;
 
 	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, ctx);
 	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, dc);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
 	first_link = dm_kunit_alloc_link(test);
 	second_link = dm_kunit_alloc_link(test);
 
 	ctx->dc = dc;
+	dc->link_srv = link_srv;
+	link_srv->dp_should_enable_fec = dm_test_dp_should_enable_fec;
+	first_link->dc = dc;
+	second_link->dc = dc;
 	dc->link_count = 2;
 	dc->links[0] = first_link;
 	dc->links[1] = second_link;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
index 6e599657dac1..a2ae30eb55bf 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
@@ -2035,14 +2035,27 @@ static void dm_test_outbox_init_null_dc(struct kunit *test)
 static void dm_test_hpd_init_empty_connectors(struct kunit *test)
 {
 	struct amdgpu_device *adev;
+	struct dc_link *link;
+	struct dc *dc;
 
 	adev = dm_kunit_alloc_adev(test);
 
 	/*
-	 * With an empty connector list the per-connector loop is skipped and
-	 * the initial clear loop relies on amdgpu_dm_irq_set() being a no-op
-	 * for a NULL dc, so init must complete without touching the DC.
+	 * The initial clear loop walks adev->dm.dc->links, so a DC with a
+	 * populated irq service is required. With an empty connector list the
+	 * per-connector loop is skipped, so init must complete after only the
+	 * clear loop runs.
 	 */
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
+	adev->dm.dc = dc;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	link->irq_source_hpd = DC_IRQ_SOURCE_HPD1;
+	link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX;
+	dc->links[0] = link;
+	dc->link_count = 1;
+
 	amdgpu_dm_hpd_init(adev);
 }
 
@@ -2070,13 +2083,17 @@ static void dm_test_hpd_init_fini_with_connectors(struct kunit *test)
 	struct drm_connector *wbconn;
 	struct amdgpu_device *adev;
 	struct dc_link *link;
+	struct dc *dc;
 
 	adev = dm_kunit_alloc_adev(test);
 
+	/* The clear loop walks dc->links, so set up a DC with an irq service. */
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
+	adev->dm.dc = dc;
+
 	/*
 	 * num_hpd = 0 forces irq_type >= num_hpd so the loop takes the HW
-	 * fallback (amdgpu_dm_irq_set()) instead of amdgpu_irq_get(); with a
-	 * NULL dc that fallback is a safe no-op.
+	 * fallback (amdgpu_dm_irq_set()) instead of amdgpu_irq_get().
 	 */
 	adev->mode_info.num_hpd = 0;
 
@@ -2102,8 +2119,10 @@ static void dm_test_hpd_init_fini_with_connectors(struct kunit *test)
 	link->irq_source_hpd = DC_IRQ_SOURCE_HPD1;
 	link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX;
 	aconn->dc_link = link;
+	dc->links[0] = link;
+	dc->link_count = 1;
 
-	/* DC is absent (NULL), so the HW writes are NULL-safe no-ops. */
+	/* The irq service stubs accept the HW writes without register access. */
 	amdgpu_dm_hpd_init(adev);
 	amdgpu_dm_hpd_fini(adev);
 }
@@ -2117,10 +2136,15 @@ static void dm_test_hpd_init_fini_analog_connector(struct kunit *test)
 	struct amdgpu_dm_connector *aconn;
 	struct amdgpu_device *adev;
 	struct dc_link *link;
+	struct dc *dc;
 
 	adev = dm_kunit_alloc_adev(test);
 	adev->mode_info.num_hpd = 0;
 
+	/* The clear loop walks dc->links, so set up a DC with an irq service. */
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
+	adev->dm.dc = dc;
+
 	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
 	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
@@ -2136,6 +2160,8 @@ static void dm_test_hpd_init_fini_analog_connector(struct kunit *test)
 	/* An analog connector id makes the loop request polling. */
 	link->link_id.id = CONNECTOR_ID_VGA;
 	aconn->dc_link = link;
+	dc->links[0] = link;
+	dc->link_count = 1;
 
 	/* use_polling becomes true, so init must enable KMS polling. */
 	amdgpu_dm_hpd_init(adev);
@@ -2155,9 +2181,14 @@ static void dm_test_hpd_init_fini_irq_ref(struct kunit *test)
 	struct amdgpu_dm_connector *aconn;
 	struct amdgpu_device *adev;
 	struct dc_link *link;
+	struct dc *dc;
 
 	adev = dm_kunit_alloc_adev(test);
 
+	/* The clear loop walks dc->links, so set up a DC with an irq service. */
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
+	adev->dm.dc = dc;
+
 	/*
 	 * num_hpd >= 1 makes irq_type (0) < num_hpd, so the loop takes the
 	 * amdgpu_irq_get()/amdgpu_irq_put() branch instead of the
@@ -2180,6 +2211,8 @@ static void dm_test_hpd_init_fini_irq_ref(struct kunit *test)
 	link->irq_source_hpd = DC_IRQ_SOURCE_HPD1;
 	link->irq_source_hpd_rx = DC_IRQ_SOURCE_INVALID;
 	aconn->dc_link = link;
+	dc->links[0] = link;
+	dc->link_count = 1;
 
 	amdgpu_dm_hpd_init(adev);
 	amdgpu_dm_hpd_fini(adev);
@@ -2200,6 +2233,9 @@ static void dm_test_hpd_init_skips_connector_without_link(struct kunit *test)
 	adev = dm_kunit_alloc_adev(test);
 	adev->mode_info.num_hpd = 0;
 
+	/* The clear loop walks dc->links, so set up a DC with an irq service. */
+	adev->dm.dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
+
 	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
 	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile
index 165f63cd82c2..39cffa0a8116 100644
--- a/drivers/gpu/drm/amd/display/dc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/Makefile
@@ -74,6 +74,7 @@ FILES += core/dc_hw_sequencer.o
 FILES += core/dc_link_enc_cfg.o
 FILES += core/dc_link_exports.o
 FILES += core/dc_resource.o
+FILES += core/dc_resource_helper.o
 FILES += core/dc_sink.o
 FILES += core/dc_stat.o
 FILES += core/dc_state.o
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c
index d500a3210eed..33f0099075c0 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c
@@ -418,8 +418,6 @@ static void dcn42b_update_clocks(struct clk_mgr *clk_mgr_base,
 			dc->debug.force_min_dcfclk_mhz = DCN42B_MIN_DCFCLK_MHZ;
 		else if (dc->debug.force_min_dcfclk_mhz > DCN42B_MAX_DCFCLK_MHZ)
 			dc->debug.force_min_dcfclk_mhz = DCN42B_MAX_DCFCLK_MHZ;
-	} else if (power_supply_is_system_supplied()) {
-		dc->debug.force_min_dcfclk_mhz = DCN42B_AC_MIN_DCFCLK_MHZ;
 	}
 
 	dcn42_update_clocks(clk_mgr_base, context, safe_to_lower);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 1143916391e7..3dbcece49188 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -446,6 +446,48 @@ static bool set_long_vtotal(struct dc *dc, struct dc_stream_state *stream, struc
 	return false;
 }
 
+/* A hardware re-init resets the OTG DRR registers while stream->adjust keeps the
+ * pre-reset values, so those values represent has to be marked stale.
+ */
+static void invalidate_drr_adjust(struct dc *dc)
+{
+	int i;
+
+	if (!dc->current_state)
+		return;
+
+	for (i = 0; i < dc->current_state->stream_count; i++)
+		if (dc->current_state->streams[i])
+			dc->current_state->streams[i]->timing_adjust_pending = true;
+}
+
+/* apply_ctx_to_hw() skips controller programming for a pipe it considers unchanged, so a
+ * commit on its own does not guarantee the OTG matches stream->adjust. Settle anything
+ * still outstanding here - dc_stream_adjust_vmin_vmax() cannot, because it short circuits
+ * when the requested v_totals are unchanged, which is permanent for a stream that is not
+ * driving DRR.
+ */
+static void program_pending_drr_adjust(struct dc *dc, struct dc_state *context)
+{
+	int i;
+
+	if (!dc->hwss.set_drr)
+		return;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (!resource_is_pipe_type(pipe, OTG_MASTER) || !pipe->stream_res.tg)
+			continue;
+
+		if (!pipe->stream->timing_adjust_pending)
+			continue;
+
+		dc_exit_ips_for_hw_access(dc);
+		dc->hwss.set_drr(&pipe, 1, pipe->stream->adjust);
+	}
+}
+
 /**
  * dc_stream_get_last_used_drr_vtotal - Looks up the pipe context of
  * dc_stream_state and gets the last VTOTAL used by DRR (Dynamic Refresh Rate)
@@ -1342,6 +1384,9 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 {
 	int i;
 	bool drr_unchanged;
+	bool exceeds_max_v_total = dc->caps.max_v_total != 0 &&
+			(adjust->v_total_max > dc->caps.max_v_total ||
+			 adjust->v_total_min > dc->caps.max_v_total);
 
 	/*
 	 * Don't adjust DRR while there's bandwidth optimizations pending to
@@ -1351,7 +1396,7 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 		if (dc->optimized_required &&
 			(stream->adjust.v_total_max != adjust->v_total_max ||
 			stream->adjust.v_total_min != adjust->v_total_min)) {
-			stream->adjust.timing_adjust_pending = true;
+			stream->timing_adjust_pending = true;
 			return false;
 		}
 	}
@@ -1359,7 +1404,7 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 	/* timing_adjust_pending tracks whether stream->adjust still describes the
 	 * OTG, so a cleared flag plus matching values means hw is already current.
 	 */
-	drr_unchanged = !stream->adjust.timing_adjust_pending &&
+	drr_unchanged = !stream->timing_adjust_pending &&
 			stream->adjust.v_total_min == adjust->v_total_min &&
 			stream->adjust.v_total_max == adjust->v_total_max &&
 			stream->adjust.v_total_mid == adjust->v_total_mid &&
@@ -1369,6 +1414,12 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 	if (drr_unchanged)
 		return true;
 
+	/* leave stream->adjust describing the OTG rather than latching
+	 * a pending adjustment that no later call can ever clear.
+	 */
+	if (exceeds_max_v_total && !adjust->allow_otg_v_count_halt)
+		return false;
+
 	dc_exit_ips_for_hw_access(dc);
 
 	stream->adjust.v_total_max = adjust->v_total_max;
@@ -1376,16 +1427,15 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 	stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num;
 	stream->adjust.v_total_min = adjust->v_total_min;
 	stream->adjust.allow_otg_v_count_halt = adjust->allow_otg_v_count_halt;
+	/* Only cleared once the new values have actually reached an OTG. */
+	stream->timing_adjust_pending = true;
 
-	if (dc->caps.max_v_total != 0 &&
-		(adjust->v_total_max > dc->caps.max_v_total || adjust->v_total_min > dc->caps.max_v_total)) {
-		if (adjust->allow_otg_v_count_halt) {
-			stream->adjust.timing_adjust_pending = false;
-			return set_long_vtotal(dc, stream, adjust);
-		}
-		/* Nothing reached hw, so stream->adjust no longer describes the OTG. */
-		stream->adjust.timing_adjust_pending = true;
-		return false;
+	if (exceeds_max_v_total) {
+		if (!set_long_vtotal(dc, stream, adjust))
+			return false;
+
+		stream->timing_adjust_pending = false;
+		return true;
 	}
 
 	for (i = 0; i < MAX_PIPES; i++) {
@@ -1395,7 +1445,7 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 			dc->hwss.set_drr(&pipe,
 					1,
 					*adjust);
-			stream->adjust.timing_adjust_pending = false;
+			stream->timing_adjust_pending = false;
 
 			if (dc->debug.visual_confirm == VISUAL_CONFIRM_BOOSTED_REFRESH_RATE) {
 				if (pipe->stream && pipe->plane_state) {
@@ -2398,6 +2448,8 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 		return result;
 	}
 
+	program_pending_drr_adjust(dc, context);
+
 	dc_trigger_sync(dc, context);
 
 	/* Full update should unconditionally be triggered when dc_commit_state_no_check is called */
@@ -2554,8 +2606,10 @@ enum dc_status dc_commit_streams(struct dc *dc, struct dc_commit_streams_params
 		return res;
 
 	if (!streams_changed(dc, params->streams, params->stream_count) &&
-			dc->current_state->power_source == params->power_source)
+			dc->current_state->power_source == params->power_source) {
+		program_pending_drr_adjust(dc, dc->current_state);
 		return res;
+	}
 
 	dc_exit_ips_for_hw_access(dc);
 
@@ -3580,11 +3634,9 @@ static void copy_stream_update_to_stream(struct dc *dc,
 
 	if (update->crtc_timing_adjust) {
 		if (stream->adjust.v_total_min != update->crtc_timing_adjust->v_total_min ||
-			stream->adjust.v_total_max != update->crtc_timing_adjust->v_total_max ||
-			stream->adjust.timing_adjust_pending)
-			update->crtc_timing_adjust->timing_adjust_pending = true;
+			stream->adjust.v_total_max != update->crtc_timing_adjust->v_total_max)
+			stream->timing_adjust_pending = true;
 		stream->adjust = *update->crtc_timing_adjust;
-		update->crtc_timing_adjust->timing_adjust_pending = false;
 	}
 
 	if (update->dpms_off)
@@ -4192,7 +4244,9 @@ static void commit_planes_do_stream_update_sequence(struct dc *dc,
 				resource_build_info_frame(pipe_ctx);
 				add_update_info_frame_sequence(&seq_state, pipe_ctx);
 
-				if (dc_is_dp_signal(pipe_ctx->stream->signal))
+				/* The step only writes DPCD when the debug flag is set, so gate emission on it here. */
+				if (dc_is_dp_signal(pipe_ctx->stream->signal) &&
+						dc->debug.enable_driver_sequence_debug)
 					hwss_add_dp_trace_source_sequence(&seq_state,
 							pipe_ctx->stream->link,
 							DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
@@ -6309,6 +6363,8 @@ void dc_set_power_state(struct dc *dc, enum dc_acpi_cm_power_state power_state)
 
 		dc->hwss.init_hw(dc);
 
+		invalidate_drr_adjust(dc);
+
 		if (dc->hwss.init_sys_ctx != NULL &&
 			dc->vm_pa_config.valid) {
 			dc->hwss.init_sys_ctx(dc->hwseq, dc, &dc->vm_pa_config);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index da98afee558f..415ad6ddb740 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -450,18 +450,12 @@ void get_hdr_visual_confirm_color(
 	uint32_t color_value = MAX_TG_COLOR_VALUE;
 	bool is_sdr = false;
 
-	/* Determine the overscan color based on the top-most (desktop) plane's context */
-	struct pipe_ctx *top_pipe_ctx  = pipe_ctx;
-
-	while (top_pipe_ctx->top_pipe != NULL)
-		top_pipe_ctx = top_pipe_ctx->top_pipe;
-
-	switch (top_pipe_ctx->plane_res.scl_data.format) {
+	switch (pipe_ctx->plane_res.scl_data.format) {
 	case PIXEL_FORMAT_ARGB2101010:
-		if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_PQ) {
+		if (pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_PQ) {
 			/* HDR10, ARGB2101010 - set border color to red */
 			color->color_r_cr = (uint16_t)color_value;
-		} else if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
+		} else if (pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
 			/* FreeSync 2 ARGB2101010 - set border color to pink */
 			color->color_r_cr = (uint16_t)color_value;
 			color->color_b_cb = (uint16_t)color_value;
@@ -469,10 +463,10 @@ void get_hdr_visual_confirm_color(
 			is_sdr = true;
 		break;
 	case PIXEL_FORMAT_FP16:
-		if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_PQ) {
+		if (pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_PQ) {
 			/* HDR10, FP16 - set border color to blue */
 			color->color_b_cb = (uint16_t)color_value;
-		} else if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
+		} else if (pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
 			/* FreeSync 2 HDR - set border color to green */
 			color->color_g_y = (uint16_t)color_value;
 		} else
@@ -762,7 +756,7 @@ void set_drr_and_clear_adjust_pending(
 
 	if (stream)
 		/* A null params resets DRR in hw, leaving stream->adjust stale. */
-		stream->adjust.timing_adjust_pending = (params == NULL);
+		stream->timing_adjust_pending = (params == NULL);
 }
 
 void get_fams2_visual_confirm_color(
@@ -1298,12 +1292,10 @@ void hwss_build_fast_sequence(struct dc *dc,
 		block_sequence[*num_steps].func = HWSS_UPDATE_INFO_FRAME;
 		(*num_steps)++;
 
-		if (dc_is_dp_signal(pipe_ctx->stream->signal)) {
-			block_sequence[*num_steps].params.dp_trace_source_sequence_params.link = pipe_ctx->stream->link;
-			block_sequence[*num_steps].params.dp_trace_source_sequence_params.source = DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME;
-			block_sequence[*num_steps].func = DP_TRACE_SOURCE_SEQUENCE;
-			(*num_steps)++;
-		}
+		/* The step only writes DPCD when the debug flag is set, so gate emission on it here. */
+		if (dc_is_dp_signal(pipe_ctx->stream->signal) && dc->debug.enable_driver_sequence_debug)
+			hwss_add_dp_trace_source_sequence(&seq_state, pipe_ctx->stream->link,
+					DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
 	}
 
 	if (dc->hwss.set_dmdata_attributes && stream->update_flags.bits.dmdata &&
@@ -2964,7 +2956,8 @@ void hwss_setup_dpp(union block_sequence_params *params)
 				EXPANSION_MODE_ZERO,
 				p->input_csc_color_matrix,
 				p->color_space,
-				NULL);
+				NULL,
+				p->upsp_mode);
 	}
 }
 
@@ -4041,22 +4034,20 @@ void hwss_dpp_reset(union block_sequence_params *params)
 
 void hwss_dpp_root_clock_control(union block_sequence_params *params)
 {
-	struct dce_hwseq *hws = params->dpp_root_clock_control_params.hws;
+	struct dccg *dccg = params->dpp_root_clock_control_params.dccg;
 	unsigned int dpp_inst = params->dpp_root_clock_control_params.dpp_inst;
 	bool clock_on = params->dpp_root_clock_control_params.clock_on;
+	bool root_clock_optimization_dpp = params->dpp_root_clock_control_params.root_clock_optimization_dpp;
 
-	if (hws->funcs.dpp_root_clock_control)
-		hws->funcs.dpp_root_clock_control(hws, dpp_inst, clock_on);
+	dccg->funcs->dpp_root_clock_control(dccg, dpp_inst, clock_on, root_clock_optimization_dpp);
 }
 
 void hwss_dc_ip_request_cntl(union block_sequence_params *params)
 {
-	struct dc *dc = params->dc_ip_request_cntl_params.dc;
+	struct dce_hwseq *hws = params->dc_ip_request_cntl_params.hws;
 	bool enable = params->dc_ip_request_cntl_params.enable;
-	struct dce_hwseq *hws = dc->hwseq;
 
-	if (hws->funcs.dc_ip_request_cntl)
-		hws->funcs.dc_ip_request_cntl(dc, enable);
+	hws->funcs.dc_ip_request_cntl(hws, enable);
 }
 
 void hwss_dccg_update_dpp_dto(union block_sequence_params *params)
@@ -4964,9 +4955,13 @@ void hwss_add_dc_ip_request_cntl(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		bool enable)
 {
+	if (!seq_state || !seq_state->steps || !seq_state->num_steps || !dc ||
+			!dc->hwseq || !dc->hwseq->funcs.dc_ip_request_cntl)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
 		seq_state->steps[*seq_state->num_steps].func = DC_IP_REQUEST_CNTL;
-		seq_state->steps[*seq_state->num_steps].params.dc_ip_request_cntl_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.dc_ip_request_cntl_params.hws = dc->hwseq;
 		seq_state->steps[*seq_state->num_steps].params.dc_ip_request_cntl_params.enable = enable;
 		(*seq_state->num_steps)++;
 	}
@@ -5431,18 +5426,59 @@ void hwss_add_dccg_set_ref_dscclk(struct block_sequence_state *seq_state,
 	}
 }
 
+bool hwss_is_dpp_root_clock_control_allowed(struct dce_hwseq *hws, bool clock_on)
+{
+	bool root_clock_optimization_dpp;
+
+	if (!hws || !hws->ctx || !hws->ctx->dc)
+		return false;
+
+	root_clock_optimization_dpp = hws->ctx->dc->debug.root_clock_optimization.bits.dpp;
+
+	if (hws->funcs.is_dpp_root_clock_control_supported)
+		return hws->funcs.is_dpp_root_clock_control_supported(
+				root_clock_optimization_dpp, clock_on);
+
+	/* Without ASIC policy only gating depends on the optimization */
+	return root_clock_optimization_dpp || clock_on;
+}
+
 void hwss_add_dpp_root_clock_control(struct block_sequence_state *seq_state,
 		struct dce_hwseq *hws,
 		unsigned int dpp_inst,
 		bool clock_on)
 {
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].func = DPP_ROOT_CLOCK_CONTROL;
-		seq_state->steps[*seq_state->num_steps].params.dpp_root_clock_control_params.hws = hws;
-		seq_state->steps[*seq_state->num_steps].params.dpp_root_clock_control_params.dpp_inst = dpp_inst;
-		seq_state->steps[*seq_state->num_steps].params.dpp_root_clock_control_params.clock_on = clock_on;
-		(*seq_state->num_steps)++;
-	}
+	struct dc *dc;
+	struct dccg *dccg;
+
+	if (!seq_state || !seq_state->steps || !seq_state->num_steps ||
+			*seq_state->num_steps >= MAX_HWSS_BLOCK_SEQUENCE_SIZE)
+		return;
+
+	if (!hws || !hws->funcs.dpp_root_clock_control || !hws->ctx || !hws->ctx->dc)
+		return;
+
+	dc = hws->ctx->dc;
+
+	if (!dc->res_pool || !dc->res_pool->dccg)
+		return;
+
+	dccg = dc->res_pool->dccg;
+
+	if (!dccg->funcs || !dccg->funcs->dpp_root_clock_control ||
+			dpp_inst >= ARRAY_SIZE(dccg->dpp_clock_gated))
+		return;
+
+	if (!hwss_is_dpp_root_clock_control_allowed(hws, clock_on))
+		return;
+
+	seq_state->steps[*seq_state->num_steps].func = DPP_ROOT_CLOCK_CONTROL;
+	seq_state->steps[*seq_state->num_steps].params.dpp_root_clock_control_params.dccg = dccg;
+	seq_state->steps[*seq_state->num_steps].params.dpp_root_clock_control_params.dpp_inst = dpp_inst;
+	seq_state->steps[*seq_state->num_steps].params.dpp_root_clock_control_params.clock_on = clock_on;
+	seq_state->steps[*seq_state->num_steps].params.dpp_root_clock_control_params.root_clock_optimization_dpp =
+			dc->debug.root_clock_optimization.bits.dpp;
+	(*seq_state->num_steps)++;
 }
 
 void hwss_add_dpp_pg_control(struct block_sequence_state *seq_state,
@@ -5679,6 +5715,7 @@ void hwss_add_dpp_setup_dpp(struct block_sequence_state *seq_state,
 		p->format = plane_state->format;
 		p->input_csc_color_matrix = plane_state->input_csc_color_matrix;
 		p->color_space = plane_state->color_space;
+		p->upsp_mode = resource_is_upsp_required(plane_state->format, plane_state->scaling_linearity);
 		(*seq_state->num_steps)++;
 	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index 187486d978da..fa44c68b3340 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -1557,25 +1557,6 @@ void resource_build_test_pattern_params(struct resource_context *res_ctx,
 	}
 }
 
-enum upsp_mode resource_is_upsp_required(enum surface_pixel_format format,
-		enum dc_scaling_linearity scaling_linearity)
-{
-	bool scaling_in_linear = (scaling_linearity == DC_SCALING_LINEARITY_LINEAR);
-	bool is_420_format = (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN &&
-			format <= SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb);
-	bool is_422_format = (format > SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb &&
-			format < SURFACE_PIXEL_FORMAT_SUBSAMPLE_END);
-
-	/* UPSP (chroma upsampling) is only needed when subsampled YUV is scaled in
-	 * linear space.
-	 */
-	if (scaling_in_linear && is_420_format)
-		return UPSP_HORIZONTAL_VERTICAL_UPSAMPLING;
-	if (scaling_in_linear && is_422_format)
-		return UPSP_HORIZONTAL_UPSAMPLING_ONLY;
-	return UPSP_BYPASS;
-}
-
 bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 {
 	const struct dc_plane_state *plane_state = pipe_ctx->plane_state;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource_helper.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource_helper.c
new file mode 100644
index 000000000000..55dfa465163f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource_helper.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "dc_types.h"
+#include "sspl/dc_spl_types.h"
+#include "resource.h"
+
+enum upsp_mode resource_is_upsp_required(enum surface_pixel_format format,
+		enum dc_scaling_linearity scaling_linearity)
+{
+	bool scaling_in_linear = (scaling_linearity == DC_SCALING_LINEARITY_LINEAR);
+	bool is_420_format = (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN &&
+		format <= SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb);
+	bool is_422_format = (format > SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb &&
+		format < SURFACE_PIXEL_FORMAT_SUBSAMPLE_END);
+
+	/* UPSP (chroma upsampling) is only needed when subsampled YUV is scaled in
+	 * linear space.
+	 */
+	if (scaling_in_linear && is_420_format)
+		return UPSP_HORIZONTAL_VERTICAL_UPSAMPLING;
+	if (scaling_in_linear && is_422_format)
+		return UPSP_HORIZONTAL_UPSAMPLING_ONLY;
+	return UPSP_BYPASS;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index d3c484f45333..f4fefa7c9ab8 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -68,7 +68,7 @@ struct dcn_dsc_reg_state;
 struct dcn_optc_reg_state;
 struct dcn_dccg_reg_state;
 
-#define DC_VER "3.2.401"
+#define DC_VER "3.2.403"
 
 /**
  * MAX_SURFACES - representative of the upper bound of surfaces that can be piped to a single CRTC
@@ -1188,6 +1188,7 @@ struct dc_debug_options {
 	bool enable_dmcub_surface_flip;
 	bool usbc_combo_phy_reset_wa;
 	bool force_fva;
+	bool force_dpg_on_flip;
 	int max_frl_rate;
 	unsigned int  force_frl_rate;
 	bool ignore_ffe;
@@ -1331,6 +1332,7 @@ struct dc_debug_options {
 	uint8_t iommu_mismatch_temp_wka;
 	bool disable_dynamic_expansion_for_test_pattern;
 	bool psr_phy_force_phy_power_down_up_level_2;
+	bool dmub_send_az_pme_restore;
 	uint32_t dml21_custom_derate_num_dpms;
 	uint32_t dml21_custom_derate_at_dpm[DML2_MAX_NUM_DPM_LVL];
 	bool override_utm_client_qc_profile;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index 0485ff5217f1..f1a0225bbf84 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -2606,3 +2606,20 @@ void dc_dmub_srv_hubbub_set_riommu_pctrl(const struct dc_context *ctx, uint32_t
 
 	dc_wake_and_execute_dmub_cmd(ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 }
+
+void dc_dmub_srv_nbif_az_pme_restore(const struct dc_context *ctx, uint32_t az_inst)
+{
+	union dmub_rb_cmd cmd;
+
+	az_inst =  (az_inst < 0x7) ? az_inst : 0; //dcnhw max aud_end point is 7
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	cmd.nbif_az_pme_restore.header.type = DMUB_CMD__NBIF_AZ_PME_RESTORE;
+	cmd.nbif_az_pme_restore.header.payload_bytes = sizeof(cmd.nbif_az_pme_restore.data);
+
+	cmd.nbif_az_pme_restore.data.az_inst = az_inst & 0xF;
+
+	/* AZ must be out of D3 before the endpoint is accessed, so wait for completion */
+	dc_wake_and_execute_dmub_cmd(ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
index 8056824d90e0..684b66dc9771 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
@@ -476,4 +476,15 @@ bool dc_dmub_srv_panel_polarity_get_polarity(struct dc_dmub_srv *dc_dmub_srv, ui
  */
 void dc_dmub_srv_hubbub_set_riommu_pctrl(const struct dc_context *ctx, uint32_t value);
 
+/**
+ * dc_dmub_srv_nbif_az_pme_restore() - Wake the AZ endpoint from D3 via DMCUB.
+ *
+ * Replaces the clk_mgr PME workaround on DCN4_2, where the NBIF AZ PME restore
+ * is performed by DMCUB on the driver's behalf. No-op on other ASICs.
+ *
+ * @ctx: dc context
+ * @az_inst: Azalia codec endpoint instance
+ */
+void dc_dmub_srv_nbif_az_pme_restore(const struct dc_context *ctx, uint32_t az_inst);
+
 #endif /* _DMUB_DC_SRV_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index b0e91b321f43..9353292edc0b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -922,7 +922,8 @@ union dpcd_dprx_feature_enumeration_list_cont_1 {
 		uint8_t AS_SDP_FIRST_HALF_LINE_OR_3840_PIXEL_CYCLE_WINDOW_NOT_SUPPORTED: 1;
 		uint8_t RESERVED0: 2;
 		uint8_t VSC_EXT_SDP_VER1_SUPPORT: 1;
-		uint8_t RESERVED1: 3;
+		uint8_t AS_SDP_VERSION_3_SUPPORTED: 1;
+		uint8_t RESERVED1: 2;
 	} bits;
 	uint8_t raw;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dc_fused_io.c b/drivers/gpu/drm/amd/display/dc/dc_fused_io.c
index 923f70724419..1c58223acce1 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_fused_io.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_fused_io.c
@@ -131,7 +131,9 @@ bool dm_atomic_write_poll_read_aux(
 	if (!link)
 		return false;
 
-	const uint32_t ddc_line = link->ddc->ddc_pin->pin_data->en;
+	/* If no DDC pins exist, then use the aux hw inst instead. Same logic as in dm_atomic_write_poll_read_i2c */
+	const uint32_t ddc_line = link->no_ddc_pin ?
+			link->aux_hw_inst : link->ddc->ddc_pin->pin_data->en;
 	union dmub_rb_cmd commands[3] = { 0 };
 	const bool converted = op_aux_convert(&commands[0], write, FUSED_REQUEST_WRITE, ddc_line)
 			&& op_aux_convert(&commands[1], poll, FUSED_REQUEST_POLL, ddc_line)
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h b/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h
index eb6e7f4043fd..9e0fe41f28a5 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hdmi_types.h
@@ -235,7 +235,8 @@ enum link_result {
 	LINK_RESULT_SUCCESS,
 	LINK_RESULT_LOWER_LINKRATE,
 	LINK_RESULT_TIMEOUT,
-	LINK_RESULT_FALLBACK
+	LINK_RESULT_FALLBACK,
+	LINK_RESULT_NO_TIMEOUT
 };
 
 enum hdmi_frl_link_rate {
@@ -274,6 +275,7 @@ struct dc_hdmi_frl_flags {
 	bool force_frl_dsc;
 	bool force_frl_max;
 	bool apply_vsdb_rcc_wa;
+	bool no_timeout;
 };
 
 struct dc_hdmi_frl_link_training_overrides {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index baf6091bb52d..eb9effb22658 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -1083,7 +1083,6 @@ struct dc_crtc_timing_adjust {
 	uint32_t v_total_mid;
 	uint32_t v_total_mid_frame_num;
 	uint32_t allow_otg_v_count_halt;
-	uint8_t timing_adjust_pending;
 };
 
 struct dtbclk_dto_params {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_memory_pool.c b/drivers/gpu/drm/amd/display/dc/dc_memory_pool.c
index 8a5a72f4cc88..e3db3a43550f 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_memory_pool.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_memory_pool.c
@@ -17,6 +17,7 @@
 #include "dc_memory_pool.h"
 
 #include <linux/atomic.h>
+#include <linux/compiler_attributes.h>
 
 enum {
 	DC_MEMORY_POOL_SENTINEL = -1,
diff --git a/drivers/gpu/drm/amd/display/dc/dc_memory_pool.h b/drivers/gpu/drm/amd/display/dc/dc_memory_pool.h
index 0ec305813d66..e0d3beb60ef4 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_memory_pool.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_memory_pool.h
@@ -19,6 +19,8 @@
 
 #include "os_types.h"
 
+#include <linux/compiler_attributes.h>
+
 /**
  * Non-resizable lock-free memory pool with fixed capacity.
  *
diff --git a/drivers/gpu/drm/amd/display/dc/dc_spl_translate.c b/drivers/gpu/drm/amd/display/dc/dc_spl_translate.c
index 76c0d62031b6..00e121bb8ef7 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_spl_translate.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_spl_translate.c
@@ -209,21 +209,27 @@ void translate_SPL_in_params_from_pipe_ctx(struct pipe_ctx *pipe_ctx, struct spl
 		}
 	}
 	// Translate linear light scaling preference
-	if (pipe_ctx->stream->ctx->dc->debug.force_lls > 0)
+	if (pipe_ctx->stream->ctx->dc->debug.force_lls > 0) {
 		spl_in->lls_pref = pipe_ctx->stream->ctx->dc->debug.force_lls;
-	else
+		spl_in->upsp_mode = pipe_ctx->plane_res.scl_data.upsp;
+	} else {
 		if ((plane_state->ctx->dce_version == DCN_VERSION_4_01) ||
 			(plane_state->ctx->dce_version == DCN_VERSION_4_2) ||
-			(plane_state->ctx->dce_version == DCN_VERSION_4_2B))
+			(plane_state->ctx->dce_version == DCN_VERSION_4_2B)) {
 			spl_in->lls_pref = LLS_PREF_DONT_CARE;
-		else
+			spl_in->upsp_mode = UPSP_BYPASS;
+		} else {
 			set_linear_light_scaling_preference(plane_state, spl_in);
-	/* Translate upsampling mode*/
-	spl_in->upsp_mode = pipe_ctx->plane_res.scl_data.upsp;
+			/* Translate upsampling mode*/
+			spl_in->upsp_mode = pipe_ctx->plane_res.scl_data.upsp;
+		}
+	}
 	/* Translate chroma subsampling offset ( cositing ) */
-	if (pipe_ctx->stream->ctx->dc->debug.force_cositing)
+	if (pipe_ctx->stream->ctx->dc->debug.force_cositing) {
 		spl_in->basic_in.cositing = pipe_ctx->stream->ctx->dc->debug.force_cositing - 1;
-	else
+		if (spl_in->upsp_mode && spl_in->basic_in.cositing == CHROMA_COSITING_NONE)
+			spl_in->basic_in.cositing = CHROMA_COSITING_TOPLEFT;
+	} else
 		spl_in->basic_in.cositing = plane_state->cositing;
 	/* Translate transfer function */
 	spl_in->basic_in.tf_type = (enum spl_transfer_func_type) plane_state->in_transfer_func.type;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index fec5e7a84bb9..7999a7d84a77 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -336,6 +336,9 @@ struct dc_stream_state {
 	enum dc_blending_linearity blending_linearity;
 	struct dc_update_scratch_space *update_scratch;
 	bool firmware_controlled_hdr_info_packet;
+
+	/* timing adjust has not been propagated to hardware yet. */
+	bool timing_adjust_pending;
 };
 
 #define ABM_LEVEL_IMMEDIATE_DISABLE 255
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c
index a71851779b37..7bfe7be2bc1f 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c
@@ -352,10 +352,12 @@ static void dccg314_set_valid_pixel_rate(
 static void dccg314_dpp_root_clock_control(
 		struct dccg *dccg,
 		unsigned int dpp_inst,
-		bool clock_on)
+		bool clock_on,
+		bool root_clock_optimization_dpp)
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
 
+	(void)root_clock_optimization_dpp;
 	if (dccg->dpp_clock_gated[dpp_inst] != clock_on)
 		return;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
index ba624f442915..6dce2a4beca2 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
@@ -395,9 +395,6 @@ static void dccg35_set_dppclk_rcg(struct dccg *dccg, int inst, bool allow_rcg)
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
 
-	if (!dccg->ctx->dc->debug.root_clock_optimization.bits.dpp && allow_rcg)
-		return;
-
 	switch (inst) {
 	case 0:
 		REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, DPPCLK0_ROOT_GATE_DISABLE, allow_rcg ? 0 : 1);
@@ -1145,7 +1142,8 @@ static void dccg35_disable_dpp_clk_new(
 	REG_SET_2(DPPCLK_DTO_PARAM[inst], 0,
 			  DPPCLK0_DTO_PHASE, 0,
 			  DPPCLK0_DTO_MODULO, 1);
-	dccg35_set_dppclk_rcg(dccg, inst, true);
+	if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp)
+		dccg35_set_dppclk_rcg(dccg, inst, true);
 }
 
 static void dccg35_disable_dscclk_new(struct dccg *dccg,
@@ -1301,7 +1299,8 @@ void dccg35_update_dpp_dto(struct dccg *dccg, int dpp_inst, int req_dppclk)
 		dcn35_set_dppclk_enable(dccg, dpp_inst, true);
 	} else {
 		dcn35_set_dppclk_enable(dccg, dpp_inst, false);
-		dccg35_set_dppclk_rcg(dccg, dpp_inst, true);
+		if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp)
+			dccg35_set_dppclk_rcg(dccg, dpp_inst, true);
 	}
 	udelay(10);
 	dccg->pipe_dppclk_khz[dpp_inst] = req_dppclk;
@@ -1868,7 +1867,8 @@ static void dccg35_set_valid_pixel_rate(
 	dccg35_set_dtbclk_dto(dccg, &dto_params);
 }
 
-void dccg35_dpp_root_clock_control(struct dccg *dccg, unsigned int dpp_inst, bool clock_on)
+void dccg35_dpp_root_clock_control(struct dccg *dccg, unsigned int dpp_inst, bool clock_on,
+		bool root_clock_optimization_dpp)
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
 
@@ -1890,7 +1890,8 @@ void dccg35_dpp_root_clock_control(struct dccg *dccg, unsigned int dpp_inst, boo
 			  DPPCLK0_DTO_PHASE, 0,
 			  DPPCLK0_DTO_MODULO, 1);
 		/*we have this in hwss: disable_plane*/
-		dccg35_set_dppclk_rcg(dccg, dpp_inst, true);
+		if (root_clock_optimization_dpp)
+			dccg35_set_dppclk_rcg(dccg, dpp_inst, true);
 	}
 
 	// wait for clock to fully ramp
@@ -2505,7 +2506,8 @@ static void dccg35_update_dpp_dto_cb(struct dccg *dccg, int dpp_inst,
 static void dccg35_dpp_root_clock_control_cb(
 	struct dccg *dccg,
 	unsigned int dpp_inst,
-	bool power_on)
+	bool power_on,
+	bool root_clock_optimization_dpp)
 {
 	if (dccg->dpp_clock_gated[dpp_inst] == power_on)
 		return;
@@ -2514,7 +2516,8 @@ static void dccg35_dpp_root_clock_control_cb(
 	 * Since clock source is not passed restore to refclock on ungate
 	 * Redundant as gating when enabled is acheived through update_dpp_dto
 	 */
-	dccg35_set_dppclk_rcg(dccg, dpp_inst, !power_on);
+	if (power_on || root_clock_optimization_dpp)
+		dccg35_set_dppclk_rcg(dccg, dpp_inst, !power_on);
 
 	dccg->dpp_clock_gated[dpp_inst] = !power_on;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h
index 58f4c1e998b3..7dbb0c90d352 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h
@@ -260,7 +260,8 @@ void dccg35_set_dpstreamclk_root_clock_gating(struct dccg *dccg, int dp_hpo_inst
 
 void dccg35_set_hdmistreamclk_root_clock_gating(struct dccg *dccg, bool enable);
 
-void dccg35_dpp_root_clock_control(struct dccg *dccg, unsigned int dpp_inst, bool clock_on);
+void dccg35_dpp_root_clock_control(struct dccg *dccg, unsigned int dpp_inst, bool clock_on,
+		bool root_clock_optimization_dpp);
 
 void dccg35_disable_symclk32_se(struct dccg *dccg, int hpo_se_inst);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
index bdf2bd388be7..b7b84152d623 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
@@ -304,7 +304,7 @@ static void dccg42_init(struct dccg *dccg)
 
 	if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp) {
 		for (i = 0; i < res_pool->pipe_count; i++) {
-			dccg35_dpp_root_clock_control(dccg, i, true);
+			dccg35_dpp_root_clock_control(dccg, i, true, true);
 		}
 	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
index 71eedb3301e4..62673dc8b04a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
@@ -45,7 +45,7 @@ static int median3(int a, int b, int c)
 	if (b > c)
 		swap(b, c);
 	if (a > b)
-		swap(b, c);
+		swap(a, b);
 
 	return b;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
index 04c5f09499ac..139e0152fcd6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
@@ -192,8 +192,8 @@ static const struct dml2_soc_bb dml2_socbb_dcn42 = {
 		.dram_clk_change_blackout_us = 36,
 		.fclk_change_blackout_us = 0,
 		.g7_ppt_blackout_us = 0,
-		.stutter_enter_plus_exit_latency_us = 14,
-		.stutter_exit_latency_us = 12,
+		.stutter_enter_plus_exit_latency_us = 15.5,
+		.stutter_exit_latency_us = 13.5,
 		.z8_stutter_enter_plus_exit_latency_us = 300,
 		.z8_stutter_exit_latency_us = 200,
 	},
@@ -224,8 +224,8 @@ static struct dml2_soc_power_management_parameters dcn42_ddr5_power_management_p
 	.dram_clk_change_blackout_us = 36,
 	.fclk_change_blackout_us = 0,
 	.g7_ppt_blackout_us = 0,
-	.stutter_enter_plus_exit_latency_us = 23.5,
-	.stutter_exit_latency_us = 21.5,
+	.stutter_enter_plus_exit_latency_us = 25.0,
+	.stutter_exit_latency_us = 23.0,
 	.z8_stutter_enter_plus_exit_latency_us = 300,
 	.z8_stutter_exit_latency_us = 200,
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
index 2d43a35c4c42..a4a77e375d9f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
@@ -12315,6 +12315,7 @@ static unsigned int dml_get_plane_idx(const struct dml2_core_internal_display_mo
 static void rq_dlg_get_wm_regs(const struct dml2_display_cfg *display_cfg, const struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_dchub_watermark_regs *wm_regs)
 {
 	double refclk_freq_in_mhz = (display_cfg->overrides.hw.dlg_ref_clk_mhz > 0) ? (double)display_cfg->overrides.hw.dlg_ref_clk_mhz : mode_lib->soc.dchub_refclk_mhz;
+	unsigned int u0_10_fixed_point_scale = 1U << 10;
 
 	wm_regs->fclk_pstate = (unsigned int)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
 	wm_regs->sr_enter = (unsigned int)(mode_lib->mp.Watermark.StutterEnterPlusExitWatermark * refclk_freq_in_mhz);
@@ -12327,9 +12328,9 @@ static void rq_dlg_get_wm_regs(const struct dml2_display_cfg *display_cfg, const
 	wm_regs->usr = (unsigned int)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
 	wm_regs->refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.UrgentLatency * refclk_freq_in_mhz);
 	wm_regs->refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.MetaTripToMemory * refclk_freq_in_mhz);
-	wm_regs->frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
-	wm_regs->frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
-	wm_regs->frac_urg_bw_mall = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthMALL * 1000);
+	wm_regs->frac_urg_bw_flip = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	wm_regs->frac_urg_bw_nom = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidth * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	wm_regs->frac_urg_bw_mall = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthMALL * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
 }
 
 static unsigned int log_and_substract_if_non_zero(unsigned int a, unsigned int subtrahend)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c
index f407afefb37f..262b8e098343 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c
@@ -5344,6 +5344,7 @@ void dcn5_adjust_pixel_clock_for_progressive_to_interlace_unit(const struct dml2
 static void dcn5_rq_dlg_get_wm_regs(const struct dml2_display_cfg *display_cfg, const struct dml2_core_internal_display_mode_lib *mode_lib, const struct dml2_utm_soc_bb *utm_soc_bb, struct dml2_dchub_watermark_regs *wm_regs)
 {
 	double refclk_freq_in_mhz = (display_cfg->overrides.hw.dlg_ref_clk_mhz > 0) ? (double)display_cfg->overrides.hw.dlg_ref_clk_mhz : utm_soc_bb->dchub_refclk_mhz;
+	unsigned int u0_10_fixed_point_scale = 1U << 10;
 
 	wm_regs->fclk_pstate = (int unsigned)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
 	wm_regs->sr_enter = (int unsigned)(mode_lib->mp.Watermark.StutterEnterPlusExitWatermark * refclk_freq_in_mhz);
@@ -5356,8 +5357,8 @@ static void dcn5_rq_dlg_get_wm_regs(const struct dml2_display_cfg *display_cfg,
 	wm_regs->usr = (int unsigned)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
 	wm_regs->refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.UrgentLatency * refclk_freq_in_mhz);
 	wm_regs->refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.MetaTripToMemory * refclk_freq_in_mhz);
-	wm_regs->frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
-	wm_regs->frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
+	wm_regs->frac_urg_bw_flip = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	wm_regs->frac_urg_bw_nom = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidth * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
 }
 
 void dcn5_get_mcif_arb_params(const struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_mcif_global_register_set *out)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c
index 6ad41846a73a..485ad96f64b1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c
@@ -1059,6 +1059,7 @@ static void dcn6_rq_dlg_get_dlg_reg(
 static void dcn6_rq_dlg_get_wm_regs(const struct dml2_display_cfg *display_cfg, const struct dml2_core_internal_display_mode_lib *mode_lib, const struct dml2_utm_soc_bb *utm_soc_bb, struct dml2_dchub_watermark_regs *wm_regs)
 {
 	double refclk_freq_in_mhz = (display_cfg->overrides.hw.dlg_ref_clk_mhz > 0) ? (double)display_cfg->overrides.hw.dlg_ref_clk_mhz : utm_soc_bb->dchub_refclk_mhz;
+	unsigned int u0_10_fixed_point_scale = 1U << 10;
 
 	wm_regs->fclk_pstate = (int unsigned)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
 	wm_regs->sr_enter = (int unsigned)(mode_lib->mp.Watermark.StutterEnterPlusExitWatermark * refclk_freq_in_mhz);
@@ -1074,8 +1075,8 @@ static void dcn6_rq_dlg_get_wm_regs(const struct dml2_display_cfg *display_cfg,
 	wm_regs->usr = (int unsigned)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
 	wm_regs->refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.UrgentLatency * refclk_freq_in_mhz);
 	wm_regs->refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.MetaTripToMemory * refclk_freq_in_mhz);
-	wm_regs->frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
-	wm_regs->frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
+	wm_regs->frac_urg_bw_flip = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	wm_regs->frac_urg_bw_nom = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidth * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
 }
 
 void dcn6_get_pipe_regs(const struct dml2_display_cfg *display_cfg,
@@ -1752,8 +1753,9 @@ void dcn6_calculate_pstate_schedule_windows(
 		det_fill_delay_otg_vlines = (int)(math_ceil(det_fill_delay_us[k] / otg_vline_time_us[k]));
 		reserved_vblank_otg_vlines = (int)(math_ceil(reserved_vblank_us[k] / otg_vline_time_us[k]));
 
-		allow_start_otg_vlines = v_blank_end[k] + det_fill_delay_otg_vlines;
-		allow_end_otg_vlines = v_blank_start[k] - (blackout_otg_vlines - reserved_vblank_otg_vlines);
+		/* for vactive only, make relative to vblank end */
+		allow_start_otg_vlines = det_fill_delay_otg_vlines;
+		allow_end_otg_vlines = v_blank_start[k] - v_blank_end[k] - (blackout_otg_vlines - reserved_vblank_otg_vlines);
 
 		/* Convert them back to time since the start of a frame */
 		allow_start_us[k] = allow_start_otg_vlines * otg_vline_time_us[k];
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.c
index aeddd113b7a1..50b7779a6ce7 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.c
@@ -700,8 +700,8 @@ static void dcn6_populate_fams2_programming(const struct dml2_core_internal_disp
 	base_programming->config.bits.is_drr = stream_descriptor->timing.drr_config.enabled;
 
 	/* from meta */
-	base_programming->otg_vline_time_ns =
-		(unsigned int)(stream_pstate_meta->otg_vline_time_us * 1000.0);
+	base_programming->otg_vline_time_ps =
+		(unsigned int)(stream_pstate_meta->otg_vline_time_us * 1000000.0);
 	base_programming->scheduling_delay_otg_vlines = (uint8_t)stream_pstate_meta->scheduling_delay_otg_vlines;
 	base_programming->contention_delay_otg_vlines = (uint8_t)stream_pstate_meta->contention_delay_otg_vlines;
 	base_programming->vline_int_ack_delay_otg_vlines = (uint8_t)stream_pstate_meta->vertical_interrupt_ack_delay_otg_vlines;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.c
index 71d14fb8c830..50ca620a4c41 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.c
@@ -126,7 +126,8 @@ bool dcn6_ms_check_scaler_support(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		plane = &display_cfg->plane_descriptors[k];
 		if (plane->composition.scaler_info.enabled == false
-				&& (dml2_core_utils_is_420(plane->pixel_format) || dml2_core_utils_is_422_planar(plane->pixel_format) || dml2_core_utils_is_422_packed(plane->pixel_format)
+			&& (((dml2_core_utils_is_420(plane->pixel_format) || dml2_core_utils_is_422_planar(plane->pixel_format)
+					|| dml2_core_utils_is_422_packed(plane->pixel_format)) && plane->composition.scaler_info.upsp_enabled == false)
 						|| plane->composition.scaler_info.plane0.h_ratio != 1.0
 						|| plane->composition.scaler_info.plane0.h_taps != 1.0
 						|| plane->composition.scaler_info.plane0.v_ratio != 1.0
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
index 54a1f9c13dbf..7fa50ccfcfd1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
@@ -898,6 +898,7 @@ bool dpmm_dcn4_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_
 	struct dml2_dchub_global_register_set *dchubbub_regs = &in_out->programming->global_regs;
 
 	double refclk_freq_in_mhz = (display_cfg->overrides.hw.dlg_ref_clk_mhz > 0) ? (double)display_cfg->overrides.hw.dlg_ref_clk_mhz : mode_lib->soc.dchub_refclk_mhz;
+	unsigned int u0_10_fixed_point_scale = 1U << 10;
 
 	/* set A */
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].fclk_pstate = (int unsigned)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
@@ -911,9 +912,9 @@ bool dpmm_dcn4_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].usr = (int unsigned)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_mall = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthMALL * 1000);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_flip = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_nom = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidth * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_mall = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthMALL * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
 
 	/* set B */
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].fclk_pstate = (int unsigned)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
@@ -927,9 +928,9 @@ bool dpmm_dcn4_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].usr = (int unsigned)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_mall = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthMALL * 1000);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_flip = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_nom = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidth * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_mall = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthMALL * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
 
 	dchubbub_regs->num_watermark_sets = 2;
 
@@ -944,6 +945,7 @@ bool dpmm_dcn42_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in
 	struct dml2_dchub_global_register_set *dchubbub_regs = &in_out->programming->global_regs;
 
 	double refclk_freq_in_mhz = (display_cfg->overrides.hw.dlg_ref_clk_mhz > 0) ? (double)display_cfg->overrides.hw.dlg_ref_clk_mhz : mode_lib->soc.dchub_refclk_mhz;
+	unsigned int u0_10_fixed_point_scale = 1U << 10;
 
 	/* set A */
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].fclk_pstate = (int unsigned)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
@@ -957,9 +959,9 @@ bool dpmm_dcn42_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].usr = (int unsigned)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_mall = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthMALL * 1000);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_flip = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_nom = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidth * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_mall = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthMALL * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
 
 	/* set B */
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B] = dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A];
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn5.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn5.c
index f6230f8f9127..e607d763f5a3 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn5.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn5.c
@@ -397,6 +397,7 @@ bool dpmm_dcn5_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_
 	struct dml2_mcif_global_register_set *mcif_regs = &in_out->programming->mcif_global_regs;
 
 	double refclk_freq_in_mhz = (display_cfg->overrides.hw.dlg_ref_clk_mhz > 0) ? (double)display_cfg->overrides.hw.dlg_ref_clk_mhz : in_out->core->utm_soc_bb->dchub_refclk_mhz;
+	unsigned int u0_10_fixed_point_scale = 1U << 10;
 
 	/* set A */
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].fclk_pstate = (int unsigned)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
@@ -410,9 +411,9 @@ bool dpmm_dcn5_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].usr = (int unsigned)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_mall = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthMALL * 1000);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_flip = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_nom = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidth * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_A].frac_urg_bw_mall = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthMALL * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
 
 	/* set B */
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].fclk_pstate = (int unsigned)(mode_lib->mp.Watermark.FCLKChangeWatermark * refclk_freq_in_mhz);
@@ -426,9 +427,9 @@ bool dpmm_dcn5_map_watermarks(struct dml2_dpmm_map_watermarks_params_in_out *in_
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].usr = (int unsigned)(mode_lib->mp.Watermark.USRRetrainingWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].refcyc_per_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
 	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].refcyc_per_meta_trip_to_mem = (unsigned int)(mode_lib->mp.Watermark.UrgentWatermark * refclk_freq_in_mhz);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_flip = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_nom = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidth * 1000);
-	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_mall = (unsigned int)(mode_lib->mp.FractionOfUrgentBandwidthMALL * 1000);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_flip = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthImmediateFlip * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_nom = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidth * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
+	dchubbub_regs->wm_regs[DML2_DCHUB_WATERMARK_SET_B].frac_urg_bw_mall = (unsigned int)math_min2(math_ceil(mode_lib->mp.FractionOfUrgentBandwidthMALL * u0_10_fixed_point_scale), u0_10_fixed_point_scale - 1);
 
 	dchubbub_regs->num_watermark_sets = 2;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn6_stage_optimizers.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn6_stage_optimizers.c
index 6d6611a6b5a6..40dcaa6588ff 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn6_stage_optimizers.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn6_stage_optimizers.c
@@ -407,14 +407,20 @@ static bool dcn6_update_worksheet_for_pstate_admissibility(struct dml2_optimizat
 		enum dml2_pstate_type pstate_type)
 {
 	const double vblank_ratio = 0.8;
-	unsigned int plane_index, stream_index;
-	double ideal_relative_disallow, ideal_disallow_time_us, disallow_time_us;
+	const double min_vblank_unreserved_ratio = 0.15;
+	const double min_vblank_unreserved_time_us = 50.0;
+	// min reserved for prefetch + 1 line lost for max vstartup
+	const unsigned int min_vblank_unreserved_otg_vlines = 4 + 1;
+
+	unsigned int plane_index, stream_index, stream_index1;
+	double ideal_disallow_time_us, disallow_time_us;
 	double extra_time_required_us;
 	double vblank_time_us;
 	double min_unreserved_vblank_time_us;
 	double min_det_fill_delay_us;
 	double delta_max_det_fill_delay_us;
 	double delta_reserved_vblank_time_us;
+	double smallest_allow_us;
 
 	int max_vactive_det_fill_delay_us;
 	long reserved_vblank_time_ns;
@@ -428,54 +434,67 @@ static bool dcn6_update_worksheet_for_pstate_admissibility(struct dml2_optimizat
 	 * configs failing pstate admissibility tend to have a large vblank to vactive ratio.
 	 */
 
-	ideal_relative_disallow = 1.0;
-	if (display_config->num_streams > 1) {
-		/* The disallow region should be less than (0.5) ^ (number of displays - 1) * total frame time
-		* Calculation assumes identical displays / timings when performing the scheduling check
-		* Make the relative disallow 5% smaller than the ideal case to add extra margin
-		*/
-		ideal_relative_disallow = (double)math_pow(0.5, (float)(display_config->num_streams - 1)) * 0.95;
-	}
-
 	for (stream_index = 0; stream_index < display_config->num_streams; stream_index++) {
 		disallow_time_us = per_stream_pstate_meta[stream_index].method_vactive.common.disallow_time_us;
-		ideal_disallow_time_us = per_stream_pstate_meta[stream_index].method_vactive.common.period_us * ideal_relative_disallow;
+		ideal_disallow_time_us = disallow_time_us;
+
+		/* find the smallest active to build the heurisitic for ideal disallow */
+		smallest_allow_us = 0.0;
+		for (stream_index1 = 0; stream_index1 < display_config->num_streams; stream_index1++) {
+			if (stream_index != stream_index1 &&
+					(smallest_allow_us == 0.0 ||
+					per_stream_pstate_meta[stream_index1].method_vactive.common.allow_time_us < smallest_allow_us)) {
+				smallest_allow_us = per_stream_pstate_meta[stream_index1].method_vactive.common.allow_time_us;
+			}
+		}
 
-		extra_time_required_us = 0.0;
 		delta_max_det_fill_delay_us = 0.0;
 		delta_reserved_vblank_time_us = 0.0;
-		if (disallow_time_us > ideal_disallow_time_us) {
-			/* minimum unreserved 15% of blank, or 50us left over for prefetch */
-			vblank_time_us = display_config->stream_descriptors[stream_index].timing.vblank_nom *
+		if (worksheet->fclk_ppt_temp_read_pstate.index > 0) {
+			if (smallest_allow_us > 0.0 && display_config->num_streams > 1) {
+				ideal_disallow_time_us = smallest_allow_us / math_pow(2.0, (float)display_config->num_streams - 2) *
+					(1.0 / (2.0 - math_pow(2.0, (float)(2 - display_config->num_streams))));
+			}
+
+			extra_time_required_us = 0.0;
+			delta_max_det_fill_delay_us = 0.0;
+			delta_reserved_vblank_time_us = 0.0;
+			if (disallow_time_us > ideal_disallow_time_us) {
+				/* minimum unreserved 15% of blank, or 50us left over for prefetch */
+				vblank_time_us = display_config->stream_descriptors[stream_index].timing.vblank_nom *
 					per_stream_pstate_meta[stream_index].otg_vline_time_us;
-			min_unreserved_vblank_time_us = math_min2(vblank_time_us * 0.15, 50);
+				min_unreserved_vblank_time_us = math_max2(
+					per_stream_pstate_meta[stream_index].otg_vline_time_us * min_vblank_unreserved_otg_vlines,
+					math_min2(vblank_time_us * min_vblank_unreserved_ratio,
+						min_vblank_unreserved_time_us));
 
-			/* minmum of blackout time (~2x VActive bandwidth) */
-			min_det_fill_delay_us = per_stream_pstate_meta[stream_index].blackout_otg_vlines *
+				/* minmum of blackout time (~2x VActive bandwidth) */
+				min_det_fill_delay_us = per_stream_pstate_meta[stream_index].blackout_otg_vlines *
 					per_stream_pstate_meta[stream_index].otg_vline_time_us;
 
-			extra_time_required_us = disallow_time_us - ideal_disallow_time_us;
+				extra_time_required_us = disallow_time_us - ideal_disallow_time_us;
 
-			/* try compressing VActive fill first */
-			delta_max_det_fill_delay_us = math_min2(extra_time_required_us * (1.0 - vblank_ratio),
+				/* try compressing VActive fill first */
+				delta_max_det_fill_delay_us = math_min2(extra_time_required_us * (1.0 - vblank_ratio),
 					math_max2(0.0,
-					per_stream_pstate_meta[stream_index].method_vactive.max_vactive_det_fill_delay_us - min_det_fill_delay_us));
-			delta_max_det_fill_delay_us = math_floor2(delta_max_det_fill_delay_us, per_stream_pstate_meta[stream_index].otg_vline_time_us);
-			extra_time_required_us -= delta_max_det_fill_delay_us;
+						per_stream_pstate_meta[stream_index].method_vactive.max_vactive_det_fill_delay_us - min_det_fill_delay_us));
+				delta_max_det_fill_delay_us = math_floor2(delta_max_det_fill_delay_us, per_stream_pstate_meta[stream_index].otg_vline_time_us);
+				extra_time_required_us -= delta_max_det_fill_delay_us;
 
-			/* try reserving VBlank */
-			delta_reserved_vblank_time_us = math_min2(extra_time_required_us,
+				/* try reserving VBlank */
+				delta_reserved_vblank_time_us = math_min2(extra_time_required_us,
 					math_max2(0.0,
-					vblank_time_us - per_stream_pstate_meta[stream_index].method_vactive.reserved_vblank_required_us - min_unreserved_vblank_time_us));
-			delta_reserved_vblank_time_us = math_floor2(delta_reserved_vblank_time_us, per_stream_pstate_meta[stream_index].otg_vline_time_us);
-			extra_time_required_us -= delta_reserved_vblank_time_us;
+						vblank_time_us - per_stream_pstate_meta[stream_index].method_vactive.reserved_vblank_required_us - min_unreserved_vblank_time_us));
+				delta_reserved_vblank_time_us = math_floor2(delta_reserved_vblank_time_us, per_stream_pstate_meta[stream_index].otg_vline_time_us);
+				extra_time_required_us -= delta_reserved_vblank_time_us;
 
-			if (extra_time_required_us > 0.0 &&
+				if (extra_time_required_us > 0.0 &&
 					per_stream_pstate_meta[stream_index].method_vactive.max_vactive_det_fill_delay_us > math_ceil2(delta_max_det_fill_delay_us + extra_time_required_us, per_stream_pstate_meta[stream_index].otg_vline_time_us)) {
-				/* final attempt to compress fill time */
-				delta_max_det_fill_delay_us = delta_max_det_fill_delay_us + extra_time_required_us;
-				delta_max_det_fill_delay_us = math_ceil2(delta_max_det_fill_delay_us, per_stream_pstate_meta[stream_index].otg_vline_time_us);
-				extra_time_required_us = 0.0;
+					/* final attempt to compress fill time */
+					delta_max_det_fill_delay_us = delta_max_det_fill_delay_us + extra_time_required_us;
+					delta_max_det_fill_delay_us = math_ceil2(delta_max_det_fill_delay_us, per_stream_pstate_meta[stream_index].otg_vline_time_us);
+					extra_time_required_us = 0.0;
+				}
 			}
 		}
 
@@ -1688,7 +1707,7 @@ static void dml2_pmo_dcn6_stage_optimizer_fclk_ppt_temp_read_pstate_init(
 		struct dml2_pmo_stage_optimizer *stage, struct dml2_optimization_worksheet *worksheet)
 {
 	DML_LOG_COMP_IF_ENTER();
-	worksheet->fclk_ppt_temp_read_pstate.is_attempted = false;
+	worksheet->fclk_ppt_temp_read_pstate.index = 0;
 	memset(&stage->func_locals->fclk_ppt_temp_read_pstate_optimize, 0,
 			sizeof(struct dml2_stage_optimizer_fclk_ppt_temp_read_pstate_optimize_locals));
 	DML_LOG_COMP_IF_EXIT();
@@ -1712,13 +1731,11 @@ static bool dml2_pmo_dcn6_stage_optimizer_fclk_ppt_temp_read_pstate_optimize_nex
 
 	DML_LOG_COMP_IF_ENTER();
 
-	if (worksheet->fclk_ppt_temp_read_pstate.is_attempted) {
+	if (worksheet->fclk_ppt_temp_read_pstate.index >= 2) {
 		DML_LOG_COMP_IF_EXIT();
 		return false;
 	}
 
-	worksheet->fclk_ppt_temp_read_pstate.is_attempted = true;
-
 	if (display_config->overrides.all_streams_blanked)
 		goto exit;
 
@@ -1761,6 +1778,8 @@ static bool dml2_pmo_dcn6_stage_optimizer_fclk_ppt_temp_read_pstate_optimize_nex
 		}
 	}
 
+	worksheet->fclk_ppt_temp_read_pstate.index++;
+
 exit:
 	DML_LOG_DEBUG("%s exit with should_continue = %s\n", __func__, modified ? "true" : "false");
 	DML_LOG_COMP_IF_EXIT();
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/alternate_pstate_shared_lib.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/alternate_pstate_shared_lib.c
index f082f058001e..ec2f2bdada6a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/alternate_pstate_shared_lib.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/alternate_pstate_shared_lib.c
@@ -346,6 +346,7 @@ static bool is_svp_dst_in_next_frame(
 	uint16_t vstartup)
 {
 	uint32_t dst_deadline;
+	uint32_t dst_deadline_frame_count;
 
 	dst_deadline = calculate_swath_deadline_dst(
 		swath_index,
@@ -355,7 +356,15 @@ static bool is_svp_dst_in_next_frame(
 		pre_hdl_delta_x1000,
 		rec_hdl_delta_x1000);
 
-	if ((swath_index < END_SWATH_REC && is_dst_in_next_frame(curr_otg_line, curr_frame_count, dst_deadline, target_frame_count, vstartup)) ||
+	// Special case for when prefetch_start (i.e., vblank_end - dst_y_prefetch) is in the previous frame,
+	// but the actual prefetch deadline is in the next frame. We must increment the frame count to ensure
+	// the swath is marked with "next frame" correctly.
+	if (swath_index == 0 && pre_swath_incl_boundary > (pre_first_hdl_x1000 / 1000) && dst_deadline < dst_line)
+		dst_deadline_frame_count = target_frame_count == MAX_FRAME_COUNT ? 0 : target_frame_count + 1;
+	else
+		dst_deadline_frame_count = target_frame_count;
+
+	if ((swath_index < END_SWATH_REC && is_dst_in_next_frame(curr_otg_line, curr_frame_count, dst_deadline, dst_deadline_frame_count, vstartup)) ||
 		(swath_index >= END_SWATH_REC && is_end_swath_in_next_frame(target_frame_count, curr_frame_count, curr_otg_line, dst_line, pre_swath_incl_boundary, vstartup))) {
 		return true;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
index dc9d4be9d7f7..6f527760071c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
@@ -643,7 +643,7 @@ struct dml2_optimization_worksheet {
 	} dcfclk_vmin;
 
 	struct {
-		bool is_attempted;
+		unsigned int index;
 	} fclk_ppt_temp_read_pstate;
 
 	/*
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c
index ef605e0a75e3..de420cdf1514 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.c
@@ -286,9 +286,11 @@ void dpp1_cnv_setup (
 		enum expansion_mode mode,
 		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space,
-		struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+		struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+		enum upsp_mode upsp_mode)
 {
 	(void)alpha_2bit_lut;
+	(void)upsp_mode;
 	uint32_t pixel_format;
 	uint32_t alpha_en;
 	enum pixel_format_description fmt ;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h
index 890d04b1ddaf..b04d5fa2cd32 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h
@@ -1501,7 +1501,8 @@ void dpp1_cnv_setup (
 		enum expansion_mode mode,
 		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space,
-		struct cnv_alpha_2bit_lut *alpha_2bit_lut);
+		struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+		enum upsp_mode upsp_mode);
 
 void dpp1_dppclk_control(
 		struct dpp *dpp_base,
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp.c
index 8d5000790904..0f3b1717563b 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn20/dcn20_dpp.c
@@ -103,8 +103,10 @@ static void dpp2_cnv_setup (
 		enum expansion_mode mode,
 		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space,
-		struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+		struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+		enum upsp_mode upsp_mode)
 {
+	(void)upsp_mode;
 	struct dcn20_dpp *dpp = TO_DCN20_DPP(dpp_base);
 	uint32_t pixel_format = 0;
 	uint32_t alpha_en = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn201/dcn201_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn201/dcn201_dpp.c
index d78ab3ffd95d..a2079db62f63 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn201/dcn201_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn201/dcn201_dpp.c
@@ -47,8 +47,10 @@ static void dpp201_cnv_setup(
 		enum expansion_mode mode,
 		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space,
-		struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+		struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+		enum upsp_mode upsp_mode)
 {
+	(void)upsp_mode;
 	struct dcn201_dpp *dpp = TO_DCN201_DPP(dpp_base);
 	uint32_t pixel_format = 0;
 	uint32_t alpha_en = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
index 2ebc738c37f2..3db10019fa84 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
@@ -100,6 +100,78 @@ void dpp30_read_reg_state(struct dpp *dpp_base, struct dcn_dpp_reg_state *dpp_re
 	dpp_reg_state->mpc_size = REG_READ(MPC_SIZE);
 }
 
+void dpp3_program_pre_csc(
+		struct dpp *dpp_base,
+		enum dc_color_space color_space,
+		enum dcn10_input_csc_select input_select,
+		const struct out_csc_color_matrix *tbl_entry)
+{
+	int i;
+	enum dcn10_input_csc_select select;
+	struct color_matrices_reg gam_regs;
+	struct dcn3_dpp *dpp = TO_DCN30_DPP(dpp_base);
+	int arr_size = sizeof(dpp_input_csc_matrix)/sizeof(struct dpp_input_csc_matrix);
+	const uint16_t *regval = NULL;
+	uint32_t cur_select = 0;
+
+	if (input_select == INPUT_CSC_SELECT_BYPASS) {
+		REG_SET(PRE_CSC_MODE, 0, PRE_CSC_MODE, 0);
+		return;
+	}
+
+	if (tbl_entry == NULL) {
+		for (i = 0; i < arr_size; i++)
+			if (dpp_input_csc_matrix[i].color_space == color_space) {
+				regval = dpp_input_csc_matrix[i].regval;
+				break;
+			}
+
+		if (regval == NULL) {
+			BREAK_TO_DEBUGGER();
+			return;
+		}
+	} else {
+		regval = tbl_entry->regval;
+	}
+
+	/* determine which CSC matrix (icsc or coma) we are using
+	 * currently.  select the alternate set to double buffer
+	 * the CSC update so CSC is updated on frame boundary
+	 */
+	REG_GET(PRE_CSC_MODE,
+			PRE_CSC_MODE_CURRENT, &cur_select);
+
+	if (cur_select != INPUT_CSC_SELECT_ICSC)
+		select = INPUT_CSC_SELECT_ICSC;
+	else
+		select = INPUT_CSC_SELECT_COMA;
+
+	gam_regs.shifts.csc_c11 = dpp->tf_shift->PRE_CSC_C11;
+	gam_regs.masks.csc_c11  = dpp->tf_mask->PRE_CSC_C11;
+	gam_regs.shifts.csc_c12 = dpp->tf_shift->PRE_CSC_C12;
+	gam_regs.masks.csc_c12 = dpp->tf_mask->PRE_CSC_C12;
+
+	if (select == INPUT_CSC_SELECT_ICSC) {
+
+		gam_regs.csc_c11_c12 = REG(PRE_CSC_C11_C12);
+		gam_regs.csc_c33_c34 = REG(PRE_CSC_C33_C34);
+
+	} else {
+
+		gam_regs.csc_c11_c12 = REG(PRE_CSC_B_C11_C12);
+		gam_regs.csc_c33_c34 = REG(PRE_CSC_B_C33_C34);
+
+	}
+
+	cm_helper_program_color_matrices(
+			dpp->base.ctx,
+			regval,
+			&gam_regs);
+
+	REG_SET(PRE_CSC_MODE, 0,
+			PRE_CSC_MODE, select);
+}
+
 /*program post scaler scs block in dpp CM*/
 void dpp3_program_post_csc(
 		struct dpp *dpp_base,
@@ -223,8 +295,10 @@ void dpp3_cnv_setup (
 		enum expansion_mode mode,
 		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space,
-		struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+		struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+		enum upsp_mode upsp_mode)
 {
+	(void)upsp_mode;
 	struct dcn3_dpp *dpp = TO_DCN30_DPP(dpp_base);
 	uint32_t pixel_format = 0;
 	uint32_t alpha_en = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h
index f397e8222ac9..349a1b9ee916 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h
@@ -629,7 +629,8 @@ void dpp3_cnv_setup(
 		enum expansion_mode mode,
 		struct dc_csc_transform input_csc_color_matrix,
 		enum dc_color_space input_color_space,
-		struct cnv_alpha_2bit_lut *alpha_2bit_lut);
+		struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+		enum upsp_mode upsp_mode);
 
 void dpp3_program_CM_bias(
 		struct dpp *dpp_base,
@@ -650,6 +651,12 @@ void dpp3_set_cursor_attributes(
 		struct dpp *dpp_base,
 		struct dc_cursor_attributes *cursor_attributes);
 
+void dpp3_program_pre_csc(
+		struct dpp *dpp_base,
+		enum dc_color_space color_space,
+		enum dcn10_input_csc_select input_select,
+		const struct out_csc_color_matrix *tbl_entry);
+
 void dpp3_program_post_csc(
 		struct dpp *dpp_base,
 		enum dc_color_space color_space,
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c
index 8e0fa7be34ee..7a3a7678c71a 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c
@@ -58,8 +58,10 @@ void dpp401_dpp_setup(
 	enum expansion_mode mode,
 	struct dc_csc_transform input_csc_color_matrix,
 	enum dc_color_space input_color_space,
-	struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+	struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+	enum upsp_mode upsp_mode)
 {
+	(void)upsp_mode;
 	struct dcn401_dpp *dpp = TO_DCN401_DPP(dpp_base);
 	uint32_t pixel_format = 0;
 	uint32_t alpha_en = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
index e34d291de37b..e49be524b31c 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
@@ -712,7 +712,8 @@ void dpp401_dpp_setup(
 	enum expansion_mode mode,
 	struct dc_csc_transform input_csc_color_matrix,
 	enum dc_color_space input_color_space,
-	struct cnv_alpha_2bit_lut *alpha_2bit_lut);
+	struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+	enum upsp_mode upsp_mode);
 
 void dpp401_set_cursor_attributes(
 	struct dpp *dpp_base,
@@ -756,10 +757,6 @@ enum dcn401_dscl_mode_sel dpp401_dscl_get_dscl_mode(
 	const struct scaler_data *data,
 	bool dbg_always_scale);
 
-void dpp401_power_on_dscl(
-	struct dpp *dpp_base,
-	bool power_on);
-
 void dpp401_dscl_set_recout(
 	struct dcn401_dpp *dpp,
 	const struct rect *recout);
@@ -792,4 +789,8 @@ void dpp401_dscl_program_easf(
 	struct dpp *dpp_base,
 	const struct scaler_data *scl_data);
 
+void dpp401_power_on_isharp(
+	struct dpp *dpp_base,
+	bool power_on);
+
 #endif /* __DCN401_DPP_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
index 81f287c89562..adb3f155d051 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
@@ -146,7 +146,7 @@ enum dcn401_dscl_mode_sel dpp401_dscl_get_dscl_mode(
 	return DCN401_DSCL_MODE_SCALING_420_YCBCR_ENABLE;
 }
 
-void dpp401_power_on_dscl(
+static void dpp401_power_on_dscl(
 	struct dpp *dpp_base,
 	bool power_on)
 {
@@ -173,6 +173,25 @@ void dpp401_power_on_dscl(
 	}
 }
 
+void dpp401_power_on_isharp(
+	struct dpp *dpp_base,
+	bool power_on)
+{
+	struct dcn401_dpp *dpp = TO_DCN401_DPP(dpp_base);
+
+	if (power_on) {
+		/* exit LS prior to programming ISHARP LUT */
+		REG_UPDATE_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+		     ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
+		     ISHARP_DELTA_LUT_MEM_PWR_DIS, 1);
+		REG_WAIT(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+			ISHARP_DELTA_LUT_MEM_PWR_STATE, 0, 1, 100);
+	} else {
+		REG_UPDATE_SEQ_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+			 ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
+			 ISHARP_DELTA_LUT_MEM_PWR_DIS, 0);
+	}
+}
 
 static void dpp401_dscl_set_lb(
 	struct dcn401_dpp *dpp,
@@ -478,8 +497,9 @@ enum lb_memory_config dpp401_dscl_find_lb_memory_config(struct dcn401_dpp *dpp,
 	int ceil_vratio_c = dc_fixpt_ceil(scl_data->ratios.vert_c);
 
 	if (dpp->base.ctx->dc->debug.use_max_lb) {
-		if (scl_data->format == PIXEL_FORMAT_420BPP8
+		if ((scl_data->format == PIXEL_FORMAT_420BPP8
 				|| scl_data->format == PIXEL_FORMAT_420BPP10)
+				&& !scl_data->upsp)
 			return LB_MEMORY_CONFIG_3;
 		return LB_MEMORY_CONFIG_0;
 	}
@@ -498,8 +518,9 @@ enum lb_memory_config dpp401_dscl_find_lb_memory_config(struct dcn401_dpp *dpp,
 			&& dpp401_dscl_is_lb_conf_valid(ceil_vratio_c, num_part_c, vtaps_c))
 		return LB_MEMORY_CONFIG_2;
 
-	if (scl_data->format == PIXEL_FORMAT_420BPP8
-			|| scl_data->format == PIXEL_FORMAT_420BPP10) {
+	if ((scl_data->format == PIXEL_FORMAT_420BPP8
+			|| scl_data->format == PIXEL_FORMAT_420BPP10)
+			&& !scl_data->upsp) {
 		dpp->base.caps->dscl_calc_lb_num_partitions(
 				scl_data, LB_MEMORY_CONFIG_3, &num_part_y, &num_part_c);
 
@@ -962,13 +983,6 @@ void dpp401_dscl_program_isharp(struct dpp *dpp_base,
 	*bs_coeffs_updated = false;
 
 	PERF_TRACE();
-	/*power on isharp_delta_mem first*/
-	REG_UPDATE_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
-		     ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
-		     ISHARP_DELTA_LUT_MEM_PWR_DIS, 1);
-
-	REG_WAIT(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
-		ISHARP_DELTA_LUT_MEM_PWR_STATE, 0, 1, 100);
 
 	/* ISHARP_MODE */
 	REG_SET_6(ISHARP_MODE, 0,
@@ -1047,11 +1061,6 @@ void dpp401_dscl_program_isharp(struct dpp *dpp_base,
 		}
 	}
 
-	/*power on isharp_delta_mem first*/
-	REG_UPDATE_SEQ_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
-			 ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
-			 ISHARP_DELTA_LUT_MEM_PWR_DIS, 0);
-
 	PERF_TRACE();
 } // dpp401_dscl_program_isharp
 /**
@@ -1094,7 +1103,10 @@ void dpp401_dscl_set_scaler_manual_scale(struct dpp *dpp_base,
 		(dpp->scl_data.dscl_prog_data.sharpness_level
 		!= scl_data->dscl_prog_data.sharpness_level)) {
 		/* ISHARP_DELTA_LUT */
+		dpp401_power_on_isharp(dpp_base, true);
 		dpp401_dscl_set_isharp_filter(dpp, scl_data->dscl_prog_data.isharp_delta);
+		dpp401_power_on_isharp(dpp_base, false);
+
 		dpp->scl_data.dscl_prog_data.sharpness_level = scl_data->dscl_prog_data.sharpness_level;
 		memcpy(dpp->scl_data.dscl_prog_data.isharp_delta, scl_data->dscl_prog_data.isharp_delta,
 			sizeof(uint32_t) * ISHARP_LUT_TABLE_SIZE);
@@ -1157,7 +1169,9 @@ void dpp401_dscl_set_scaler_manual_scale(struct dpp *dpp_base,
 	if (dscl_mode == DCN401_DSCL_MODE_SCALING_444_BYPASS) {
 		if (dpp->base.ctx->dc->config.prefer_easf)
 			dpp401_dscl_disable_easf(dpp_base, scl_data);
+		dpp401_power_on_isharp(dpp_base, true);
 		dpp401_dscl_program_isharp(dpp_base, scl_data, program_isharp_1dlut, &bs_coeffs_updated);
+		dpp401_power_on_isharp(dpp_base, false);
 		return;
 	}
 
@@ -1191,7 +1205,9 @@ void dpp401_dscl_set_scaler_manual_scale(struct dpp *dpp_base,
 	 *   WB scaler coeffs and toggle coeff RAM together
 	 */
 	//if (dpp->base.ctx->dc->config.prefer_easf)
+	dpp401_power_on_isharp(dpp_base, true);
 	dpp401_dscl_program_isharp(dpp_base, scl_data, program_isharp_1dlut, &bs_coeffs_updated);
+	dpp401_power_on_isharp(dpp_base, false);
 
 	dpp401_dscl_set_scl_filter(dpp, scl_data, ycbcr, bs_coeffs_updated);
 	/* Edge adaptive scaler function configuration */
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
index eed86f5ae731..2ce0694e3725 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
@@ -118,8 +118,10 @@ static void dpp42_dpp_setup(
 	enum expansion_mode mode,
 	struct dc_csc_transform input_csc_color_matrix,
 	enum dc_color_space input_color_space,
-	struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+	struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+	enum upsp_mode upsp_mode)
 {
+	(void)upsp_mode;
 	struct dcn401_dpp *dpp = TO_DCN401_DPP(dpp_base);
 	uint32_t pixel_format = 0;
 	uint32_t alpha_en = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c
index 38739ed048fa..5382fc656169 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c
@@ -90,8 +90,10 @@ void dpp50_dpp_setup(
 	enum expansion_mode mode,
 	struct dc_csc_transform input_csc_color_matrix,
 	enum dc_color_space input_color_space,
-	struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+	struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+	enum upsp_mode upsp_mode)
 {
+	(void)upsp_mode;
 	struct dcn50_dpp *dpp = TO_DCN50_DPP(dpp_base);
 	uint32_t pixel_format = 0;
 	uint32_t alpha_en = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h
index c81e910ae095..fb9ed75ae901 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h
@@ -77,7 +77,8 @@ void dpp50_dpp_setup(
 	enum expansion_mode mode,
 	struct dc_csc_transform input_csc_color_matrix,
 	enum dc_color_space input_color_space,
-	struct cnv_alpha_2bit_lut *alpha_2bit_lut);
+	struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+	enum upsp_mode upsp_mode);
 
 void dpp50_set_pregam_state(
 	struct dpp *dpp_base,
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c
index be2effb14fc3..2ba8218e2ac2 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c
@@ -24,7 +24,8 @@ void dpp60_dpp_setup(
 	enum expansion_mode mode,
 	struct dc_csc_transform input_csc_color_matrix,
 	enum dc_color_space input_color_space,
-	struct cnv_alpha_2bit_lut *alpha_2bit_lut)
+	struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+	enum upsp_mode upsp_mode)
 {
 	struct dcn60_dpp *dpp = TO_DCN60_DPP(dpp_base);
 	uint32_t pixel_format = 8;
@@ -249,25 +250,47 @@ void dpp60_dpp_setup(
 		PRE_REALPHA_EN, realpha_en,
 		PRE_REALPHA_ABLND_EN, realpha_ablnd_en);
 
-	if (format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN ||
-			format >= SURFACE_PIXEL_FORMAT_SUBSAMPLE_END ||
-			color_space != COLOR_SPACE_SRGB) {
-		/* If input adjustment exists, program the ICSC with those values. */
+	// color matrix applied PreCSC for upsp mode
+	if (upsp_mode) {
 		if (input_csc_color_matrix.enable_adjustment == true) {
 			for (i = 0; i < 12; i++)
 				tbl_entry.regval[i] = input_csc_color_matrix.matrix[i];
 
 			tbl_entry.color_space = input_color_space;
 
+			// Re-use post csc function to program pre csc as
+			// the programming is same for both
 			if (dpp3_should_bypass_post_csc_for_colorspace(color_space))
 				select = INPUT_CSC_SELECT_BYPASS;
 			else
 				select = INPUT_CSC_SELECT_ICSC;
 
-			dpp3_program_post_csc(dpp_base, color_space, select,
+			dpp3_program_pre_csc(dpp_base, color_space, select,
 				&tbl_entry);
 		} else {
-			dpp3_program_post_csc(dpp_base, color_space, select, NULL);
+			dpp3_program_pre_csc(dpp_base, color_space, select, NULL);
+		}
+	} else {
+		if (format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN ||
+			format >= SURFACE_PIXEL_FORMAT_SUBSAMPLE_END ||
+			color_space != COLOR_SPACE_SRGB) {
+			/* If input adjustment exists, program the ICSC with those values. */
+			if (input_csc_color_matrix.enable_adjustment == true) {
+				for (i = 0; i < 12; i++)
+					tbl_entry.regval[i] = input_csc_color_matrix.matrix[i];
+
+				tbl_entry.color_space = input_color_space;
+
+				if (dpp3_should_bypass_post_csc_for_colorspace(color_space))
+					select = INPUT_CSC_SELECT_BYPASS;
+				else
+					select = INPUT_CSC_SELECT_ICSC;
+
+				dpp3_program_post_csc(dpp_base, color_space,
+						select, &tbl_entry);
+			} else {
+				dpp3_program_post_csc(dpp_base, color_space, select, NULL);
+			}
 		}
 	}
 }
@@ -293,10 +316,25 @@ void dpp60_full_bypass(struct dpp *dpp_base)
 		REG_SET(CM_CONTROL, 0, CM_BYPASS, 1);
 }
 
+void dcn6_dpp_reset(struct dpp *dpp_base)
+{
+	struct dcn60_dpp *dpp = TO_DCN60_DPP(dpp_base);
+
+	if (dpp->scl_data.upsp) {
+		if (dpp->tf_mask->UPSP_MODE) {
+			// Reset dpp registers used for upsp, lls
+			REG_WRITE(CNVC_UPSP_MODE, 0);
+			REG_WRITE(PRE_CSC_MODE, 0);
+		}
+	}
+
+	dpp_reset(dpp_base);
+}
+
 static struct dpp_funcs dcn60_dpp_funcs = {
 	.dpp_program_gamcor_lut		= dpp6_program_gamcor_lut,
 	.dpp_read_state				= dpp401_read_state,
-	.dpp_reset					= dpp_reset,
+	.dpp_reset					= dcn6_dpp_reset,
 	.dpp_set_scaler				= dpp60_dscl_set_scaler_manual_scale,
 	.dpp_get_optimal_number_of_taps	= dpp3_get_optimal_number_of_taps,
 	.dpp_set_pre_degam			= NULL,
@@ -317,6 +355,7 @@ static struct dpp_funcs dcn60_dpp_funcs = {
 	.dpp_cm_hist_read			= dpp42_dpp_cm_hist_read,
 	.dpp_read_reg_state			= dpp30_read_reg_state,
 	.dpp_set_pregam_state		= dpp50_set_pregam_state,
+	.dpp_program_upsp			= dpp60_dscl_program_upsp
 };
 
 static struct dpp_caps dcn60_dpp_cap = {
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h
index 5d09fcd04553..7086ff773de8 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h
@@ -448,12 +448,12 @@
 #define DPP_REG_VARIABLE_LIST_DCN60 \
 	DPP_REG_VARIABLE_LIST_DCN50; \
 	uint32_t SCL_BLACK_COLOR; \
-	uint32_t UPSP_MODE; \
-	uint32_t UPSP_V_COEF_P0; \
-	uint32_t UPSP_V_COEF_P1; \
-	uint32_t UPSP_H_COEF_P0; \
-	uint32_t UPSP_H_COEF_P1; \
-	uint32_t UPSP_CLAMP
+	uint32_t CNVC_UPSP_MODE; \
+	uint32_t CNVC_UPSP_V_COEF_P0; \
+	uint32_t CNVC_UPSP_V_COEF_P1; \
+	uint32_t CNVC_UPSP_H_COEF_P0; \
+	uint32_t CNVC_UPSP_H_COEF_P1; \
+	uint32_t CNVC_UPSP_CLAMP
 
 struct dcn60_dpp_registers {
 	DPP_REG_VARIABLE_LIST_DCN60;
@@ -490,7 +490,8 @@ void dpp60_dpp_setup(
 	enum expansion_mode mode,
 	struct dc_csc_transform input_csc_color_matrix,
 	enum dc_color_space input_color_space,
-	struct cnv_alpha_2bit_lut *alpha_2bit_lut);
+	struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+	enum upsp_mode upsp_mode);
 
 void dpp60_full_bypass(struct dpp *dpp_base);
 
@@ -521,5 +522,6 @@ bool dpp60_construct(struct dcn60_dpp *dpp60,
 bool dpp6_program_gamcor_lut(
 	struct dpp *dpp_base, const struct pwl_params *params);
 
+void dcn6_dpp_reset(struct dpp *dpp_base);
 
 #endif /* __DCN60_DPP_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp_dscl.c
index 0898148ba704..d1a81e5616b3 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp_dscl.c
@@ -166,7 +166,10 @@ void dpp60_dscl_set_scaler_manual_scale(struct dpp *dpp_base,
 		(dpp->scl_data.dscl_prog_data.sharpness_level
 		!= scl_data->dscl_prog_data.sharpness_level)) {
 		/* ISHARP_DELTA_LUT */
+		dpp401_power_on_isharp(dpp_base, true);
 		dpp401_dscl_set_isharp_filter(dpp401, scl_data->dscl_prog_data.isharp_delta);
+		dpp401_power_on_isharp(dpp_base, false);
+
 		dpp->scl_data.dscl_prog_data.sharpness_level = scl_data->dscl_prog_data.sharpness_level;
 		memcpy(dpp->scl_data.dscl_prog_data.isharp_delta, scl_data->dscl_prog_data.isharp_delta,
 			sizeof(uint32_t) * ISHARP_LUT_TABLE_SIZE);
@@ -187,6 +190,7 @@ void dpp60_dscl_set_scaler_manual_scale(struct dpp *dpp_base,
 		v_num_taps_c = scl_data->dscl_prog_data.taps.v_taps_c;
 		h_num_taps = scl_data->dscl_prog_data.taps.h_taps;
 		h_num_taps_c = scl_data->dscl_prog_data.taps.h_taps_c;
+		ycbcr = ycbcr && !scl_data->dscl_prog_data.upsp_mode;
 	}
 
 	/* Unconditionally power on DSCL - can be in light sleep otherwise. */
@@ -229,7 +233,9 @@ void dpp60_dscl_set_scaler_manual_scale(struct dpp *dpp_base,
 	if (dscl_mode == DCN401_DSCL_MODE_SCALING_444_BYPASS) {
 		if (dpp->base.ctx->dc->config.prefer_easf)
 			dpp401_dscl_disable_easf(dpp_base, scl_data);
+		dpp401_power_on_isharp(dpp_base, true);
 		dpp401_dscl_program_isharp(dpp_base, scl_data, program_isharp_1dlut, &bs_coeffs_updated);
+		dpp401_power_on_isharp(dpp_base, false);
 		return;
 	}
 
@@ -260,7 +266,9 @@ void dpp60_dscl_set_scaler_manual_scale(struct dpp *dpp_base,
 	 *   WB scaler coeffs and toggle coeff RAM together
 	 */
 	//if (dpp->base.ctx->dc->config.prefer_easf)
+	dpp401_power_on_isharp(dpp_base, true);
 	dpp401_dscl_program_isharp(dpp_base, scl_data, program_isharp_1dlut, &bs_coeffs_updated);
+	dpp401_power_on_isharp(dpp_base, false);
 
 	dpp401_dscl_set_scl_filter(dpp401, scl_data, ycbcr, bs_coeffs_updated);
 	/* Edge adaptive scaler function configuration */
@@ -280,7 +288,7 @@ void dpp60_dscl_program_upsp(struct dpp *dpp_base,
 {
 	struct dcn60_dpp *dpp = TO_DCN60_DPP(dpp_base);
 
-	REG_SET_8(UPSP_MODE, 0,
+	REG_SET_8(CNVC_UPSP_MODE, 0,
 			UPSP_MODE, dscl_prog_data->upsp_mode,
 			UPSP_V_NUM_TAPS, dscl_prog_data->upsp_v_num_taps,
 			UPSP_V_INIT_INT, dscl_prog_data->upsp_v_init_int,
@@ -289,27 +297,27 @@ void dpp60_dscl_program_upsp(struct dpp *dpp_base,
 			UPSP_H_INIT_INT, dscl_prog_data->upsp_h_init_int,
 			UPSP_H_INIT_FRAC, dscl_prog_data->upsp_h_init_frac,
 			UPSP_BOUNDARY_MODE, dscl_prog_data->upsp_boundary_mode);
-	REG_SET_4(UPSP_V_COEF_P0, 0,
+	REG_SET_4(CNVC_UPSP_V_COEF_P0, 0,
 			UPSP_V_COEF_TAP0_P0, dscl_prog_data->upsp_v_coef_tap0_p0,
 			UPSP_V_COEF_TAP1_P0, dscl_prog_data->upsp_v_coef_tap1_p0,
 			UPSP_V_COEF_TAP2_P0, dscl_prog_data->upsp_v_coef_tap2_p0,
 			UPSP_V_COEF_TAP3_P0, dscl_prog_data->upsp_v_coef_tap3_p0);
-	REG_SET_4(UPSP_V_COEF_P1, 0,
+	REG_SET_4(CNVC_UPSP_V_COEF_P1, 0,
 			UPSP_V_COEF_TAP0_P1, dscl_prog_data->upsp_v_coef_tap0_p1,
 			UPSP_V_COEF_TAP1_P1, dscl_prog_data->upsp_v_coef_tap1_p1,
 			UPSP_V_COEF_TAP2_P1, dscl_prog_data->upsp_v_coef_tap2_p1,
 			UPSP_V_COEF_TAP3_P1, dscl_prog_data->upsp_v_coef_tap3_p1);
-	REG_SET_4(UPSP_H_COEF_P0, 0,
+	REG_SET_4(CNVC_UPSP_H_COEF_P0, 0,
 			UPSP_H_COEF_TAP0_P0, dscl_prog_data->upsp_h_coef_tap0_p0,
 			UPSP_H_COEF_TAP1_P0, dscl_prog_data->upsp_h_coef_tap1_p0,
 			UPSP_H_COEF_TAP2_P0, dscl_prog_data->upsp_h_coef_tap2_p0,
 			UPSP_H_COEF_TAP3_P0, dscl_prog_data->upsp_h_coef_tap3_p0);
-	REG_SET_4(UPSP_H_COEF_P1, 0,
+	REG_SET_4(CNVC_UPSP_H_COEF_P1, 0,
 			UPSP_H_COEF_TAP0_P1, dscl_prog_data->upsp_h_coef_tap0_p1,
 			UPSP_H_COEF_TAP1_P1, dscl_prog_data->upsp_h_coef_tap1_p1,
 			UPSP_H_COEF_TAP2_P1, dscl_prog_data->upsp_h_coef_tap2_p1,
 			UPSP_H_COEF_TAP3_P1, dscl_prog_data->upsp_h_coef_tap3_p1);
-	REG_SET_2(UPSP_CLAMP, 0,
+	REG_SET_2(CNVC_UPSP_CLAMP, 0,
 			UPSP_CLAMP_MAX, dscl_prog_data->upsp_clamp_max,
 			UPSP_CLAMP_MIN, dscl_prog_data->upsp_clamp_min);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
index 8dd4b7b9ba36..786843362061 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
@@ -838,9 +838,12 @@ static bool hubbub31_get_dcc_compression_cap(struct hubbub *hubbub,
 		else
 			dcc_control = dcc_control__256_64_64;
 	} else if (input->scan == SCAN_DIRECTION_VERTICAL) {
-		if (!req128_vert_wc)
-			dcc_control = dcc_control__256_256_xxx;
-		else if (segment_order_vert == segment_order__contiguous)
+		if (!req128_vert_wc) {
+			if (segment_order_vert == segment_order__non_contiguous)
+				dcc_control = dcc_control__256_64_64;
+			else
+				dcc_control = dcc_control__256_256_xxx;
+		} else if (segment_order_vert == segment_order__contiguous)
 			dcc_control = dcc_control__128_128_xxx;
 		else
 			dcc_control = dcc_control__256_64_64;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
index 7b3aa02dda63..a08ceaeda91c 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
@@ -60,6 +60,7 @@
 #include "dpcd_defs.h"
 #include "dsc.h"
 #include "dc_dp_types.h"
+#include "dc_dmub_srv.h"
 /* include DCE11 register header files */
 #include "dce/dce_11_0_d.h"
 #include "dce/dce_11_0_sh_mask.h"
@@ -101,6 +102,7 @@ struct dce110_hw_seq_reg_offsets {
 	uint32_t crtc;
 };
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static const struct dce110_hw_seq_reg_offsets reg_offsets[] = {
 {
 	.crtc = (mmCRTC0_CRTC_GSL_CONTROL - mmCRTC_GSL_CONTROL),
@@ -121,6 +123,7 @@ static const struct dce110_hw_seq_reg_offsets reg_offsets[] = {
 
 #define HW_REG_CRTC(reg, id)\
 	(reg + reg_offsets[id].crtc)
+#endif
 
 #define MAX_WATERMARK 0xFFFF
 #define SAFE_NBP_MARK 0x7FFF
@@ -129,6 +132,7 @@ static const struct dce110_hw_seq_reg_offsets reg_offsets[] = {
  * Private definitions
  ******************************************************************************/
 /***************************PIPE_CONTROL***********************************/
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static void dce110_init_pte(struct dc_context *ctx)
 {
 	uint32_t addr;
@@ -195,6 +199,7 @@ static void dce110_init_pte(struct dc_context *ctx)
 		dm_write_reg(ctx, addr, value);
 	}
 }
+#endif
 /**************************************************************************/
 
 static void enable_display_pipe_clock_gating(
@@ -206,6 +211,7 @@ static void enable_display_pipe_clock_gating(
 	/*TODO*/
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static bool dce110_enable_display_power_gating(
 	struct dc *dc,
 	uint8_t controller_id,
@@ -630,6 +636,8 @@ dce110_set_output_transfer_func(struct set_output_transfer_func_params *params)
 	return true;
 }
 
+#endif
+
 void dce110_update_info_frame(struct pipe_ctx *pipe_ctx)
 {
 	bool is_hdmi_tmds;
@@ -697,6 +705,7 @@ dce110_external_encoder_control(enum bp_external_encoder_control_action action,
 		DC_LOG_ERROR("Failed to execute external encoder action: 0x%x\n", action);
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static void
 dce110_prepare_ddc(struct dc_link *link)
 {
@@ -716,6 +725,8 @@ dce110_dac_load_detect(struct dc_link *link)
 	return bp_result == BP_RESULT_OK;
 }
 
+#endif
+
 void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
 {
 	enum dc_lane_count lane_count =
@@ -1146,7 +1157,10 @@ void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx)
 			if (dc->current_state->res_ctx.pipe_ctx[i].stream_res.audio != NULL)
 				num_audio++;
 		}
-		if (num_audio >= 1 && clk_mgr->funcs->enable_pme_wa) {
+		if (dc->debug.dmub_send_az_pme_restore) {
+			/*wake AZ from D3 first before access az endpoint*/
+			dc_dmub_srv_nbif_az_pme_restore(dc->ctx, pipe_ctx->stream_res.audio->inst);
+		} else if (num_audio >= 1 && clk_mgr->funcs->enable_pme_wa) {
 			/*wake AZ from D3 first before access az endpoint*/
 			clk_mgr->funcs->enable_pme_wa(clk_mgr);
 		}
@@ -1184,10 +1198,13 @@ void dce110_disable_audio_stream(struct pipe_ctx *pipe_ctx)
 	if (pipe_ctx->stream_res.audio) {
 		pipe_ctx->stream_res.audio->enabled = false;
 
-		if (clk_mgr->funcs->enable_pme_wa)
-			/*this is the first audio. apply the PME w/a in order to wake AZ from D3*/
+		if (dc->debug.dmub_send_az_pme_restore) {
+			/*wake AZ from D3 first before access az endpoint*/
+			dc_dmub_srv_nbif_az_pme_restore(dc->ctx, pipe_ctx->stream_res.audio->inst);
+		} else if (clk_mgr->funcs->enable_pme_wa) {
+			/*wake AZ from D3 first before access az endpoint*/
 			clk_mgr->funcs->enable_pme_wa(clk_mgr);
-
+		}
 		/* TODO: notify audio driver for if audio modes list changed
 		 * add audio mode list change flag */
 		/* dal_audio_disable_azalia_audio_jack_presence(stream->audio,
@@ -1285,8 +1302,6 @@ void dce110_blank_stream(struct pipe_ctx *pipe_ctx)
 		return;
 
 	if (link->local_sink && link->local_sink->sink_signal == SIGNAL_TYPE_EDP) {
-		if (link->forced_psr_active)
-			return;
 		if (!link->skip_implict_edp_power_control && hws)
 			hws->funcs.edp_backlight_control(link, false);
 		link->dc->hwss.set_abm_immediate_disable(pipe_ctx);
@@ -1590,6 +1605,7 @@ void build_audio_output(
 	}
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static void program_scaler(const struct dc *dc,
 		const struct pipe_ctx *pipe_ctx)
 {
@@ -1695,6 +1711,8 @@ static enum dc_status dce110_enable_stream_timing(
 	return DC_OK;
 }
 
+#endif
+
 enum dc_status dce110_apply_single_controller_ctx_to_hw(
 		struct pipe_ctx *pipe_ctx,
 		struct dc_state *context,
@@ -1815,9 +1833,7 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw(
 			dc->link_srv->set_dsc_enable(pipe_ctx, true);
 	}
 
-	if (!stream->dpms_off &&
-	    !(link->connector_signal == SIGNAL_TYPE_EDP &&
-	      link->forced_psr_active))
+	if (!stream->dpms_off)
 		dc->link_srv->set_dpms_on(context, pipe_ctx);
 
 	/* DCN3.1 FPGA Workaround
@@ -2281,6 +2297,7 @@ void dce110_set_safe_displaymarks(
  * Public functions
  ******************************************************************************/
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static void set_drr(struct pipe_ctx **pipe_ctx,
 		int num_pipes, struct dc_crtc_timing_adjust adjust)
 {
@@ -2355,6 +2372,8 @@ static void set_static_screen_control(struct pipe_ctx **pipe_ctx,
 					triggers, params->num_frames);
 }
 
+#endif
+
 /*
  *  Check if FBC can be enabled
  */
@@ -2447,6 +2466,7 @@ void enable_fbc(
 	}
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static void dce110_reset_hw_ctx_wrap(
 		struct dc *dc,
 		struct dc_state *context)
@@ -2519,6 +2539,8 @@ static void dce110_reset_hw_ctx_wrap(
 	}
 }
 
+#endif
+
 static void dce110_setup_audio_dto(
 		struct dc *dc,
 		struct dc_state *context)
@@ -2761,6 +2783,7 @@ enum dc_status dce110_apply_ctx_to_hw(
 /*******************************************************************************
  * Front End programming
  ******************************************************************************/
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static void set_default_colors(struct pipe_ctx *pipe_ctx)
 {
 	struct default_adjustment default_adjust = { 0 };
@@ -2890,12 +2913,15 @@ static void dce110_update_pending_status(struct pipe_ctx *pipe_ctx)
 	}
 }
 
+#endif
+
 void dce110_power_down(struct dc *dc)
 {
 	power_down_all_hw_blocks(dc);
 	disable_vga_and_power_gate_all_controllers(dc);
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static bool wait_for_reset_trigger_to_occur(
 	struct dc_context *dc_ctx,
 	struct timing_generator *tg)
@@ -3102,6 +3128,8 @@ static void dce110_init_hw(struct dc *dc)
 }
 
 
+#endif
+
 void dce110_prepare_bandwidth(
 		struct dc *dc,
 		struct dc_state *context)
@@ -3131,6 +3159,7 @@ void dce110_optimize_bandwidth(
 				true);
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static void dce110_program_front_end_for_pipe(
 		struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
@@ -3408,6 +3437,8 @@ static void dce110_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
 				pipe_ctx->plane_res.xfm, attributes);
 }
 
+#endif
+
 bool dce110_set_backlight_level(struct pipe_ctx *pipe_ctx,
 	struct set_backlight_level_params *backlight_level_params)
 {
@@ -3511,6 +3542,7 @@ void dce110_enable_tmds_link_output(struct dc_link *link,
 				link->ddc_hw_inst, false);
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static void dce110_enable_analog_link_output(
 		struct dc_link *link,
 		uint32_t pix_clk_100hz)
@@ -3520,6 +3552,8 @@ static void dce110_enable_analog_link_output(
 			pix_clk_100hz);
 }
 
+#endif
+
 void dce110_enable_dp_link_output(
 		struct dc_link *link,
 		const struct link_resource *link_res,
@@ -3607,11 +3641,8 @@ void dce110_disable_link_output(struct dc_link *link,
 	else if (dmcu != NULL && dmcu->funcs->lock_phy)
 		dmcu->funcs->lock_phy(dmcu);
 
-	if (!(signal == SIGNAL_TYPE_EDP &&
-			link->forced_psr_active)) {
-		link_hwss->disable_link_output(link, link_res, signal);
-		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
-	}
+	link_hwss->disable_link_output(link, link_res, signal);
+	link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
 	/*
 	 * Add the logic to extract BOTH power up and power down sequences
 	 * from enable/disable link output and only call edp panel control
@@ -3622,6 +3653,7 @@ void dce110_disable_link_output(struct dc_link *link,
 	dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static const struct hw_sequencer_funcs dce110_funcs = {
 	.program_gamut_remap = program_gamut_remap,
 	.program_output_csc = program_output_csc,
@@ -3688,8 +3720,8 @@ static const struct hwseq_private_funcs dce110_private_funcs = {
 
 void dce110_hw_sequencer_construct(struct dc *dc)
 {
-	if (IS_ENABLED(CONFIG_DRM_AMD_DC_DCE)) {
-		dc->hwss = dce110_funcs;
-		dc->hwseq->funcs = dce110_private_funcs;
-	}
+	dc->hwss = dce110_funcs;
+	dc->hwseq->funcs = dce110_private_funcs;
 }
+
+#endif /* CONFIG_DRM_AMD_DC_DCE */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
index 771555ec1431..f1d14b97977f 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
@@ -1059,7 +1059,8 @@ static void power_on_plane_resources(
 {
 	DC_LOGGER_INIT(hws->ctx);
 
-	if (hws->funcs.dpp_root_clock_control)
+	if (hws->funcs.dpp_root_clock_control &&
+	    hwss_is_dpp_root_clock_control_allowed(hws, true))
 		hws->funcs.dpp_root_clock_control(hws, plane_id, true);
 
 	if (REG(DC_IP_REQUEST_CNTL)) {
@@ -1349,12 +1350,7 @@ static void dcn10_reset_back_end_for_pipe(
 	 * screen only, the dpms_off would be true but
 	 * VBIOS lit up eDP, so check link status too.
 	 */
-	if (link->connector_signal == SIGNAL_TYPE_EDP &&
-	    link->forced_psr_active) {
-		/* forced psr is active for seamless switch; skip dpms-off. */
-		if (pipe_ctx->stream_res.audio)
-			dc->hwss.disable_audio_stream(pipe_ctx);
-	} else if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
+	if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
 		dc->link_srv->set_dpms_off(pipe_ctx);
 	else if (pipe_ctx->stream_res.audio)
 		dc->hwss.disable_audio_stream(pipe_ctx);
@@ -1378,15 +1374,13 @@ static void dcn10_reset_back_end_for_pipe(
 	 * parent pipe.
 	 */
 	if (pipe_ctx->top_pipe == NULL) {
-		if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
-			link->forced_psr_active)) {
-			if (pipe_ctx->stream_res.abm)
-				dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
-			pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+		if (pipe_ctx->stream_res.abm)
+			dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
-			pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
-		}
+		pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+
+		pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
 		set_drr_and_clear_adjust_pending(pipe_ctx, pipe_ctx->stream, NULL);
 		if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal))
 			pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0;
@@ -1566,7 +1560,8 @@ void dcn10_plane_atomic_power_down(struct dc *dc,
 				"Power gated front end %d\n", hubp->inst);
 	}
 
-	if (hws->funcs.dpp_root_clock_control)
+	if (hws->funcs.dpp_root_clock_control &&
+	    hwss_is_dpp_root_clock_control_allowed(hws, false))
 		hws->funcs.dpp_root_clock_control(hws, dpp->inst, false);
 }
 
@@ -2941,7 +2936,8 @@ static void dcn10_update_dpp(struct dpp *dpp, struct dc_plane_state *plane_state
 			EXPANSION_MODE_ZERO,
 			plane_state->input_csc_color_matrix,
 			plane_state->color_space,
-			NULL);
+			NULL,
+			0);
 
 	//set scale and bias registers
 	build_prescale_params(&bns_params, plane_state);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
index e8b9495c799f..636cbd947918 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
@@ -1249,6 +1249,13 @@ void dcn20_blank_pixel_data(
 	/* get opp dpg blank color */
 	color_space_to_black_color(dc, color_space, &black_color);
 
+	/* Debug: force a solid blue DPG pattern. */
+	if (dc->debug.force_dpg_on_flip) {
+		black_color.color_r_cr = 0;
+		black_color.color_g_y = 0;
+		black_color.color_b_cb = 0x3ff;
+	}
+
 	if (blank) {
 		dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
@@ -1303,7 +1310,8 @@ static void dcn20_power_on_plane_resources(
 
 	DC_LOGGER_INIT(hws->ctx->logger);
 
-	if (hws->funcs.dpp_root_clock_control)
+	if (hws->funcs.dpp_root_clock_control &&
+	    hwss_is_dpp_root_clock_control_allowed(hws, true))
 		hws->funcs.dpp_root_clock_control(hws, pipe_ctx->plane_res.dpp->inst, true);
 
 	if (REG(DC_IP_REQUEST_CNTL)) {
@@ -1786,7 +1794,8 @@ void dcn20_update_dchubp_dpp(
 				EXPANSION_MODE_ZERO,
 				plane_state->input_csc_color_matrix,
 				plane_state->color_space,
-				NULL);
+				NULL,
+				pipe_ctx->plane_res.scl_data.upsp);
 
 		if (dpp->funcs->set_cursor_matrix) {
 			dpp->funcs->set_cursor_matrix(dpp,
@@ -2912,12 +2921,7 @@ void dcn20_reset_back_end_for_pipe(
 	 * screen only, the dpms_off would be true but
 	 * VBIOS lit up eDP, so check link status too.
 	 */
-	if (link->connector_signal == SIGNAL_TYPE_EDP &&
-	    link->forced_psr_active) {
-		/* forced psr is active for seamless switch; skip dpms-off. */
-		if (pipe_ctx->stream_res.audio)
-			dc->hwss.disable_audio_stream(pipe_ctx);
-	} else if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
+	if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
 		dc->link_srv->set_dpms_off(pipe_ctx);
 	else if (pipe_ctx->stream_res.audio)
 		dc->hwss.disable_audio_stream(pipe_ctx);
@@ -2942,14 +2946,12 @@ void dcn20_reset_back_end_for_pipe(
 	 * parent pipe.
 	 */
 	if (pipe_ctx->top_pipe == NULL) {
-		if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
-				link->forced_psr_active)) {
-			dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
-			pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+		dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
-			pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
-		}
+		pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+
+		pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
 		if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
 			pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
 					pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
index 823ef06c64f5..b6f3c0480ab6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
@@ -537,11 +537,9 @@ static void dcn31_reset_back_end_for_pipe(
 	}
 	ASSERT(!pipe_ctx->top_pipe);
 
-	link = pipe_ctx->stream->link;
+	dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
-	if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
-	      link->forced_psr_active))
-		dc->hwss.set_abm_immediate_disable(pipe_ctx);
+	link = pipe_ctx->stream->link;
 
 	if (dc->hwseq)
 		dc->hwseq->wa_state.skip_blank_stream = false;
@@ -557,11 +555,9 @@ static void dcn31_reset_back_end_for_pipe(
 			pipe_ctx->stream_res.tg,
 			OPTC_DSC_DISABLED, 0, 0);
 
-	if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
-	      link->forced_psr_active)) {
-		pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
-		pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
-	}
+	pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+
+	pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
 	if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
 		pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
 				pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
@@ -590,12 +586,7 @@ static void dcn31_reset_back_end_for_pipe(
 	 * screen only, the dpms_off would be true but
 	 * VBIOS lit up eDP, so check link status too.
 	 */
-	if (link->connector_signal == SIGNAL_TYPE_EDP &&
-	    link->forced_psr_active) {
-		/* forced psr is active for seamless switch; skip dpms-off. */
-		if (pipe_ctx->stream_res.audio)
-			dc->hwss.disable_audio_stream(pipe_ctx);
-	} else if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
+	if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
 		dc->link_srv->set_dpms_off(pipe_ctx);
 	else if (pipe_ctx->stream_res.audio)
 		dc->hwss.disable_audio_stream(pipe_ctx);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
index 93d8483d6f80..de7dde2b036b 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
@@ -465,14 +465,18 @@ void dcn314_resync_fifo_dccg_dio(struct dce_hwseq *hws, struct dc *dc, struct dc
 	}
 }
 
-void dcn314_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on)
+bool dcn314_is_dpp_root_clock_control_supported(bool root_clock_optimization_dpp, bool clock_on)
 {
-	if (!hws->ctx->dc->debug.root_clock_optimization.bits.dpp)
-		return;
+	(void)clock_on;
+	return root_clock_optimization_dpp;
+}
 
+void dcn314_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on)
+{
 	if (hws->ctx->dc->res_pool->dccg->funcs->dpp_root_clock_control)
 		hws->ctx->dc->res_pool->dccg->funcs->dpp_root_clock_control(
-			hws->ctx->dc->res_pool->dccg, dpp_inst, clock_on);
+			hws->ctx->dc->res_pool->dccg, dpp_inst, clock_on,
+			hws->ctx->dc->debug.root_clock_optimization.bits.dpp);
 }
 
 static void apply_symclk_on_tx_off_wa(struct dc_link *link)
@@ -527,11 +531,9 @@ void dcn314_disable_link_output(struct dc_link *link,
 		link->dc->hwss.edp_backlight_control(link, false);
 	else if (dmcu != NULL && dmcu->funcs->lock_phy)
 		dmcu->funcs->lock_phy(dmcu);
-	if (!(signal == SIGNAL_TYPE_EDP &&
-		link->forced_psr_active)) {
-		link_hwss->disable_link_output(link, link_res, signal);
-		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
-	}
+
+	link_hwss->disable_link_output(link, link_res, signal);
+	link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
 	/*
 	 * Add the logic to extract BOTH power up and power down sequences
 	 * from enable/disable link output and only call edp panel control
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.h
index 6c072d0274ea..e33237fb74be 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.h
@@ -45,6 +45,8 @@ void dcn314_resync_fifo_dccg_dio(struct dce_hwseq *hws, struct dc *dc, struct dc
 
 void dcn314_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on);
 
+bool dcn314_is_dpp_root_clock_control_supported(bool root_clock_optimization_dpp, bool clock_on);
+
 void dcn314_disable_link_output(struct dc_link *link, const struct link_resource *link_res, enum signal_type signal);
 
 void dcn314_dpp_pg_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool power_on);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
index 2b04598b0935..69b020e2c4b2 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
@@ -145,6 +145,7 @@ static const struct hwseq_private_funcs dcn314_private_funcs = {
 	.plane_atomic_power_down = dcn10_plane_atomic_power_down,
 	.enable_power_gating_plane = dcn314_enable_power_gating_plane,
 	.dpp_root_clock_control = dcn314_dpp_root_clock_control,
+	.is_dpp_root_clock_control_supported = dcn314_is_dpp_root_clock_control_supported,
 	.hubp_pg_control = dcn31_hubp_pg_control,
 	.dpp_pg_control = dcn314_dpp_pg_control,
 	.program_all_writeback_pipes_in_tree = dcn30_program_all_writeback_pipes_in_tree,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
index e7bf865755d4..9135b01fe388 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
@@ -1463,11 +1463,8 @@ void dcn32_disable_link_output(struct dc_link *link,
 	else if (dmcu != NULL && dmcu->funcs->lock_phy)
 		dmcu->funcs->lock_phy(dmcu);
 
-	if (!(signal == SIGNAL_TYPE_EDP &&
-		link->forced_psr_active)) {
-		link_hwss->disable_link_output(link, link_res, signal);
-		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
-	}
+	link_hwss->disable_link_output(link, link_res, signal);
+	link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
 	/*
 	 * Add the logic to extract BOTH power up and power down sequences
 	 * from enable/disable link output and only call edp panel control
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
index 23c09801f12f..aa047b5b4939 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
@@ -496,14 +496,17 @@ void dcn35_update_odm(struct dc *dc, struct dc_state *context, struct pipe_ctx *
 	}
 }
 
-void dcn35_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on)
+bool dcn35_is_dpp_root_clock_control_supported(bool root_clock_optimization_dpp, bool clock_on)
 {
-	if (!hws->ctx->dc->debug.root_clock_optimization.bits.dpp && !clock_on)
-		return;
+	return root_clock_optimization_dpp || clock_on;
+}
 
+void dcn35_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on)
+{
 	if (hws->ctx->dc->res_pool->dccg->funcs->dpp_root_clock_control) {
 		hws->ctx->dc->res_pool->dccg->funcs->dpp_root_clock_control(
-			hws->ctx->dc->res_pool->dccg, dpp_inst, clock_on);
+			hws->ctx->dc->res_pool->dccg, dpp_inst, clock_on,
+			hws->ctx->dc->debug.root_clock_optimization.bits.dpp);
 	}
 }
 
@@ -1365,7 +1368,8 @@ void dcn35_root_clock_control(struct dc *dc,
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
 				update_state->pg_pipe_res_update[PG_DPP][i]) {
-				if (dc->hwseq->funcs.dpp_root_clock_control)
+				if (dc->hwseq->funcs.dpp_root_clock_control &&
+				    hwss_is_dpp_root_clock_control_allowed(dc->hwseq, power_on))
 					dc->hwseq->funcs.dpp_root_clock_control(dc->hwseq, i, power_on);
 			}
 			if (update_state->pg_pipe_res_update[PG_DPSTREAM][i])
@@ -1398,7 +1402,8 @@ void dcn35_root_clock_control(struct dc *dc,
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
 				update_state->pg_pipe_res_update[PG_DPP][i]) {
-				if (dc->hwseq->funcs.dpp_root_clock_control)
+				if (dc->hwseq->funcs.dpp_root_clock_control &&
+				    hwss_is_dpp_root_clock_control_allowed(dc->hwseq, power_on))
 					dc->hwseq->funcs.dpp_root_clock_control(dc->hwseq, i, power_on);
 			}
 			if (update_state->pg_pipe_res_update[PG_DPSTREAM][i])
@@ -1844,11 +1849,8 @@ void dcn35_disable_link_output(struct dc_link *link,
 		disable_link_output_symclk_on_tx_off(link, DP_UNKNOWN_ENCODING);
 		link->phy_state.symclk_state = SYMCLK_ON_TX_OFF;
 	} else {
-		if (!(signal == SIGNAL_TYPE_EDP &&
-		      link->forced_psr_active)) {
-			link_hwss->disable_link_output(link, link_res, signal);
-			link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
-		}
+		link_hwss->disable_link_output(link, link_res, signal);
+		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
 	}
 	/*
 	 * Add the logic to extract BOTH power up and power down sequences
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
index 96b836b2ae71..0ae2f44cfb9f 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
@@ -39,6 +39,8 @@ void dcn35_dsc_pg_control(struct dce_hwseq *hws, unsigned int dsc_inst, bool pow
 
 void dcn35_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on);
 
+bool dcn35_is_dpp_root_clock_control_supported(bool root_clock_optimization_dpp, bool clock_on);
+
 void dcn35_dpstream_root_clock_control(struct dce_hwseq *hws, unsigned int dp_hpo_inst, bool clock_on);
 
 void dcn35_hdmistream_root_clock_control(struct dce_hwseq *hws, bool clock_on);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
index f8c8ab12bdf9..2119a8e000e9 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
@@ -163,6 +163,7 @@ static const struct hwseq_private_funcs dcn35_private_funcs = {
 	//.plane_atomic_disable = dcn20_plane_atomic_disable,/*todo*/
 	//.hubp_pg_control = dcn35_hubp_pg_control,
 	.dpp_root_clock_control = dcn35_dpp_root_clock_control,
+	.is_dpp_root_clock_control_supported = dcn35_is_dpp_root_clock_control_supported,
 	.dpstream_root_clock_control = dcn35_dpstream_root_clock_control,
 	.hdmistream_root_clock_control = dcn35_hdmistream_root_clock_control,
 	.physymclk_root_clock_control = dcn35_physymclk_root_clock_control,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
index 82a8d87f9428..0e148c6e5c50 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
@@ -152,6 +152,7 @@ static const struct hwseq_private_funcs dcn351_private_funcs = {
 	//.plane_atomic_disable = dcn20_plane_atomic_disable,/*todo*/
 	//.hubp_pg_control = dcn35_hubp_pg_control,
 	.dpp_root_clock_control = dcn35_dpp_root_clock_control,
+	.is_dpp_root_clock_control_supported = dcn35_is_dpp_root_clock_control_supported,
 	.dpstream_root_clock_control = dcn35_dpstream_root_clock_control,
 	.hdmistream_root_clock_control = dcn35_hdmistream_root_clock_control,
 	.physymclk_root_clock_control = dcn35_physymclk_root_clock_control,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
index 0abdfc55290c..f2391d45d4c5 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
@@ -818,6 +818,15 @@ enum dc_status dcn401_enable_stream_timing(
 
 	hws->funcs.blank_pixel_data(dc, pipe_ctx, true);
 
+	/* Legacy code path will enable DSC during DPMS on sequence. Since DSC
+	 * HW block programming is independent from link and stream encoder operations, it
+	 * has moved to hwss and should be called when enabling/disabling OTG.
+	 */
+	if (stream->timing.flags.DSC && pipe_ctx->stream_res.dsc) {
+		if (dc->hwss.configure_dsc)
+			dc->hwss.configure_dsc(pipe_ctx, true);
+	}
+
 	/* VTG is  within DCHUB command block. DCFCLK is always on */
 	if (false == pipe_ctx->stream_res.tg->funcs->enable_crtc(pipe_ctx->stream_res.tg)) {
 		BREAK_TO_DEBUGGER();
@@ -2116,6 +2125,15 @@ void dcn401_reset_back_end_for_pipe(
 
 			pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
 
+			/* Legacy code path will enable DSC during DPMS on sequence. Since DSC
+			 * HW block programming is independent from backend operations, it
+			 * has moved to hwss and should be called when enabling/disabling OTG.
+			 */
+			if (pipe_ctx->stream_res.dsc) {
+				if (dc->hwss.configure_dsc)
+					dc->hwss.configure_dsc(pipe_ctx, false);
+			}
+
 			pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
 		}
 		if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
@@ -2705,6 +2723,17 @@ void dcn401_program_front_end_for_ctx(
 			pipe->plane_res.hubp->funcs->hubp_wait_pipe_read_start(pipe->plane_res.hubp);
 		}
 	}
+
+	/* Debug (DM SystemDisplayEnable, driver disabled): force solid-color DPG
+	 * output last, after all normal pipe/OPP programming, so it isn't overwritten.
+	 */
+	if (dc->debug.force_dpg_on_flip && hws->funcs.blank_pixel_data) {
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			pipe = &context->res_ctx.pipe_ctx[i];
+			if (resource_is_pipe_type(pipe, OTG_MASTER))
+				hws->funcs.blank_pixel_data(dc, pipe, true);
+		}
+	}
 }
 
 void dcn401_post_unlock_program_front_end(
@@ -3130,7 +3159,8 @@ void dcn401_plane_atomic_power_down(struct dc *dc,
 	DC_LOG_DEBUG(
 			"Power gated front end %d\n", hubp->inst);
 
-	if (hws->funcs.dpp_root_clock_control)
+	if (hws->funcs.dpp_root_clock_control &&
+	    hwss_is_dpp_root_clock_control_allowed(hws, false))
 		hws->funcs.dpp_root_clock_control(hws, dpp->inst, false);
 }
 
@@ -3598,10 +3628,10 @@ void dcn401_post_unlock_reset_opp_sequence(
 	}
 }
 
-void dcn401_dc_ip_request_cntl(struct dc *dc, bool enable)
+void dcn401_dc_ip_request_cntl(struct dce_hwseq *hws, bool enable)
 {
-	struct dce_hwseq *hws = dc->hwseq;
-
+	if (!hws || !hws->ctx || !hws->regs || !hws->shifts || !hws->masks)
+		return;
 	if (REG(DC_IP_REQUEST_CNTL))
 		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, enable ? 1 : 0);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
index ecaeb192bca3..9df72e980e7a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
@@ -185,7 +185,7 @@ void dcn401_post_unlock_reset_opp_sequence(
 		struct pipe_ctx *opp_head,
 		struct block_sequence_state *seq_state);
 
-void dcn401_dc_ip_request_cntl(struct dc *dc, bool enable);
+void dcn401_dc_ip_request_cntl(struct dce_hwseq *hws, bool enable);
 
 void dcn401_enable_plane_sequence(struct dc *dc, struct pipe_ctx *pipe_ctx,
 				 struct dc_state *context,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
index 34bfe62296e6..b1cc09df5e63 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
@@ -1117,7 +1117,8 @@ void dcn42_root_clock_control(struct dc *dc,
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
 				update_state->pg_pipe_res_update[PG_DPP][i]) {
-				if (dc->hwseq->funcs.dpp_root_clock_control)
+				if (dc->hwseq->funcs.dpp_root_clock_control &&
+				    hwss_is_dpp_root_clock_control_allowed(dc->hwseq, power_on))
 					dc->hwseq->funcs.dpp_root_clock_control(dc->hwseq, i, power_on);
 			}
 			if (update_state->pg_pipe_res_update[PG_DPSTREAM][i])
@@ -1150,7 +1151,8 @@ void dcn42_root_clock_control(struct dc *dc,
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
 				update_state->pg_pipe_res_update[PG_DPP][i]) {
-				if (dc->hwseq->funcs.dpp_root_clock_control)
+				if (dc->hwseq->funcs.dpp_root_clock_control &&
+				    hwss_is_dpp_root_clock_control_allowed(dc->hwseq, power_on))
 					dc->hwseq->funcs.dpp_root_clock_control(dc->hwseq, i, power_on);
 			}
 			if (update_state->pg_pipe_res_update[PG_DPSTREAM][i])
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
index 775da50e945e..6711283afcd7 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
@@ -163,6 +163,7 @@ static const struct hwseq_private_funcs dcn42_private_funcs = {
 	.reset_back_end_for_pipe = dcn401_reset_back_end_for_pipe,
 	.program_cm_hist = dcn42_program_cm_hist,
 	.dpp_root_clock_control = dcn35_dpp_root_clock_control,
+	.is_dpp_root_clock_control_supported = dcn35_is_dpp_root_clock_control_supported,
 	.dpstream_root_clock_control = dcn35_dpstream_root_clock_control,
 	.hdmistream_root_clock_control = dcn35_hdmistream_root_clock_control,
 	.physymclk_root_clock_control = dcn35_physymclk_root_clock_control,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c
index 7a82aaa6d1d6..151ec838f613 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c
@@ -153,7 +153,8 @@ void dcn50_update_dchubp_dpp(
 			EXPANSION_MODE_ZERO,
 			plane_state->input_csc_color_matrix,
 			plane_state->color_space,
-			NULL);
+			NULL,
+			pipe_ctx->plane_res.scl_data.upsp);
 
 		if (dpp->funcs->set_cursor_matrix) {
 			dpp->funcs->set_cursor_matrix(dpp,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
index d515b9cbc934..c5fab430b7f7 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
@@ -53,6 +53,103 @@
 #define FN(reg_name, field_name) \
 	hws->shifts->field_name, hws->masks->field_name
 
+bool dcn60_configure_dsc(struct pipe_ctx *pipe_ctx, bool enable)
+{
+	struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
+	struct dc *dc = pipe_ctx->stream->ctx->dc;
+	struct dc_stream_state *stream = pipe_ctx->stream;
+	struct pipe_ctx *odm_pipe;
+	struct dccg *dccg = dc->res_pool->dccg;
+	struct dsc_config dsc_cfg = {0};
+	struct dsc_optc_config dsc_optc_cfg = {0};
+	uint32_t odm_count = 1;
+
+	/* It has been found that when DSCCLK is lower than 16Mhz, we will get DCN
+	 * register access hung. When DSCCLk is based on refclk, DSCCLk is always a
+	 * fixed value higher than 16Mhz so the issue doesn't occur. When DSCCLK is
+	 * generated by DTO, DSCCLK would be based on 1/3 dispclk. For small timings
+	 * with DSC such as 480p60Hz, the dispclk could be low enough to trigger
+	 * this problem. We are implementing a workaround here to keep using dscclk
+	 * based on fixed value refclk when timing is smaller than 3x16Mhz (i.e
+	 * 48Mhz) pixel clock to avoid hitting this problem.
+	 */
+	bool use_dto = dccg->funcs->set_dto_dscclk && stream->timing.pix_clk_100hz > 480000;
+
+	if (!dsc)
+		return false;
+
+	for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
+		odm_count++;
+
+	if (enable) {
+		dsc_cfg.pic_width = (stream->timing.h_addressable +
+				pipe_ctx->dsc_padding_params.dsc_hactive_padding +
+				stream->timing.h_border_left + stream->timing.h_border_right) / odm_count;
+		dsc_cfg.pic_height = stream->timing.v_addressable +
+				stream->timing.v_border_top + stream->timing.v_border_bottom;
+		dsc_cfg.pixel_encoding = stream->timing.pixel_encoding;
+		dsc_cfg.color_depth = stream->timing.display_color_depth;
+		dsc_cfg.is_odm = pipe_ctx->next_odm_pipe != NULL;
+		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
+		ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % odm_count == 0);
+		dsc_cfg.dc_dsc_cfg.num_slices_h /= odm_count;
+
+		if (use_dto)
+			dccg->funcs->set_dto_dscclk(dccg, dsc->inst,
+					(uint32_t)dsc_cfg.dc_dsc_cfg.num_slices_h);
+		if (!dsc->funcs->dsc_prepare_config(dsc, &dsc_cfg, &dsc_optc_cfg)) {
+			ASSERT(false);
+			return false;
+		}
+		dsc->funcs->dsc_set_config(dsc);
+		dsc->funcs->dsc_enable(dsc, (int)pipe_ctx->stream_res.opp->inst);
+
+		for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) {
+			struct display_stream_compressor *odm_dsc = odm_pipe->stream_res.dsc;
+
+			ASSERT(odm_dsc);
+			if (!odm_dsc)
+				return false;
+			if (use_dto)
+				dccg->funcs->set_dto_dscclk(dccg, odm_dsc->inst,
+						(uint32_t)dsc_cfg.dc_dsc_cfg.num_slices_h);
+			if (!odm_dsc->funcs->dsc_prepare_config(odm_dsc, &dsc_cfg, &dsc_optc_cfg)) {
+				ASSERT(false);
+				return false;
+			}
+			odm_dsc->funcs->dsc_set_config(odm_dsc);
+			odm_dsc->funcs->dsc_enable(odm_dsc, (int)odm_pipe->stream_res.opp->inst);
+		}
+
+		{
+			enum optc_dsc_mode mode = dsc_optc_cfg.is_pixel_format_444 ?
+					OPTC_DSC_ENABLED_444 : OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED;
+
+			DC_LOG_DSC("Setting optc DSC config for tg instance %d:", pipe_ctx->stream_res.tg->inst);
+			pipe_ctx->stream_res.tg->funcs->set_dsc_config(
+					pipe_ctx->stream_res.tg, mode,
+					dsc_optc_cfg.bytes_per_pixel, dsc_optc_cfg.slice_width);
+		}
+		return true;
+	}
+
+	DC_LOG_DSC("Setting optc DSC config for tg instance %d:", pipe_ctx->stream_res.tg->inst);
+	pipe_ctx->stream_res.tg->funcs->set_dsc_config(
+			pipe_ctx->stream_res.tg, OPTC_DSC_DISABLED, 0, 0);
+
+	for (odm_pipe = pipe_ctx; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) {
+		struct display_stream_compressor *odm_dsc = odm_pipe->stream_res.dsc;
+
+		if (!odm_dsc)
+			continue;
+		odm_dsc->funcs->dsc_disconnect(odm_dsc);
+		odm_dsc->funcs->dsc_disable(odm_dsc);
+		if (dccg->funcs->set_ref_dscclk)
+			dccg->funcs->set_ref_dscclk(dccg, (uint32_t)odm_dsc->inst);
+	}
+	return true;
+}
+
 bool dcn60_set_rmcm_luts(struct set_input_transfer_func_params *params)
 {
 	struct dpp *dpp_base = params->dpp;
@@ -413,9 +510,7 @@ enum dc_status dcn60_apply_single_controller_ctx_to_hw(
 			dc_is_virtual_signal(pipe_ctx->stream->signal)))
 			dc->link_srv->set_dsc_enable(pipe_ctx, true);
 	}
-	if (!stream->dpms_off &&
-	    !(link->connector_signal == SIGNAL_TYPE_EDP &&
-	      link->forced_psr_active))
+	if (!stream->dpms_off)
 		dc->link_srv->set_dpms_on(context, pipe_ctx);
 
 	/* DCN3.1 FPGA Workaround
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h
index b880988ff1fc..44eb30608feb 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h
@@ -31,5 +31,6 @@ void dcn60_program_perfmon(struct dc *dc, struct dc_state *context);
 bool dcn60_apply_idle_power_optimizations(struct dc *dc, bool enable);
 
 bool dcn60_set_rmcm_luts(struct set_input_transfer_func_params *params);
+bool dcn60_configure_dsc(struct pipe_ctx *pipe_ctx, bool enable);
 
 #endif /* __DC_HWSS_DCN60_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c
index 80125c3596ce..7b488753fe50 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c
@@ -25,6 +25,7 @@ static const struct hw_sequencer_funcs dcn60_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn50_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.configure_dsc = dcn60_configure_dsc,
 	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
index 7800b0501c39..8f4ed80385fc 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
@@ -155,6 +155,7 @@ struct setup_dpp_params {
 	enum surface_pixel_format format;
 	struct dc_csc_transform input_csc_color_matrix;
 	enum dc_color_space color_space;
+	enum upsp_mode upsp_mode;
 };
 
 struct program_bias_and_scale_params {
@@ -705,13 +706,14 @@ struct dpp_reset_params {
 };
 
 struct dpp_root_clock_control_params {
-	struct dce_hwseq *hws;
+	struct dccg *dccg;
 	unsigned int dpp_inst;
 	bool clock_on;
+	bool root_clock_optimization_dpp;
 };
 
 struct dc_ip_request_cntl_params {
-	struct dc *dc;
+	struct dce_hwseq *hws;
 	bool enable;
 };
 
@@ -1411,6 +1413,7 @@ struct hw_sequencer_funcs {
 			struct dc_state *context);
 	void (*update_plane_addr)(const struct dc *dc,
 			struct pipe_ctx *pipe_ctx);
+	bool (*configure_dsc)(struct pipe_ctx *pipe_ctx, bool enable);
 	void (*prepare_plane_addr_update)(const struct dc *dc,
 			struct pipe_ctx *pipe_ctx,
 			struct dc_plane_address *addr_to_program,
@@ -2455,6 +2458,8 @@ void hwss_add_dccg_set_ref_dscclk(struct block_sequence_state *seq_state,
 		int dsc_inst,
 		bool *is_ungated);
 
+bool hwss_is_dpp_root_clock_control_allowed(struct dce_hwseq *hws, bool clock_on);
+
 void hwss_add_dpp_root_clock_control(struct block_sequence_state *seq_state,
 		struct dce_hwseq *hws,
 		unsigned int dpp_inst,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
index 9586029dcec1..9c5e9abf5a95 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
@@ -135,6 +135,9 @@ struct hwseq_private_funcs {
 			struct dce_hwseq *hws,
 			unsigned int dpp_inst,
 			bool clock_on);
+	bool (*is_dpp_root_clock_control_supported)(
+			bool root_clock_optimization_dpp,
+			bool clock_on);
 	void (*dpstream_root_clock_control)(
 			struct dce_hwseq *hws,
 			unsigned int dpp_inst,
@@ -225,7 +228,7 @@ struct hwseq_private_funcs {
 			struct hubp *primary_hubp);
 	void (*wait_for_pipe_update_if_needed)(struct dc *dc, struct pipe_ctx *pipe_ctx, bool is_surface_update_only);
 	void (*set_wait_for_update_needed_for_pipe)(struct dc *dc, struct pipe_ctx *pipe_ctx);
-	void (*dc_ip_request_cntl)(struct dc *dc, bool enable);
+	void (*dc_ip_request_cntl)(struct dce_hwseq *hws, bool enable);
 	void (*program_cm_hist)(struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
 		const struct dc_plane_state *plane_state);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
index 1fe703e866fc..8538b16c9748 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
@@ -314,10 +314,8 @@ struct dccg_funcs {
 	void (*trigger_dio_fifo_resync)(
 			struct dccg *dccg);
 
-	void (*dpp_root_clock_control)(
-			struct dccg *dccg,
-			unsigned int dpp_inst,
-			bool clock_on);
+	void (*dpp_root_clock_control)(struct dccg *dccg, unsigned int dpp_inst, bool clock_on,
+			bool root_clock_optimization_dpp);
 
 	void (*enable_symclk_se)(
 			struct dccg *dccg,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index 71244d3b6c90..562176e8d7d1 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -314,7 +314,8 @@ struct dpp_funcs {
 			enum expansion_mode mode,
 			struct dc_csc_transform input_csc_color_matrix,
 			enum dc_color_space input_color_space,
-			struct cnv_alpha_2bit_lut *alpha_2bit_lut);
+			struct cnv_alpha_2bit_lut *alpha_2bit_lut,
+			enum upsp_mode upsp_mode);
 
 	void (*dpp_full_bypass)(struct dpp *dpp_base);
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_frl.c b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_frl.c
index a2c0fa21ee94..62288133cd55 100644
--- a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_frl.c
+++ b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_frl.c
@@ -59,6 +59,9 @@ static void disable_hpo_frl_link_output(struct dc_link *link,
 		enum signal_type signal)
 {
 	(void)link_res;
+	if (link->frl_flags.no_timeout)
+		return;
+
 	if (dc_is_hdmi_frl_signal(signal))
 		link->hpo_frl_link_enc->funcs->disable_link_encoder(link->hpo_frl_link_enc);
 	link->link_enc->funcs->disable_output(link->link_enc, signal);
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
index 2aeb9a0e6e80..9b5b0599d977 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
@@ -717,21 +717,94 @@ static bool dp_set_hblank_reduction_on_rx(struct pipe_ctx *pipe_ctx)
 	return result;
 }
 
+/* TODO - There is overlapping functionality between link_set_dsc_pps_packet,
+ * and set_encoder_dsc_config, the two functions should be refactored to avoid code duplication.
+ */
+static void set_encoder_dsc_config(struct pipe_ctx *pipe_ctx,
+		struct dsc_config *dsc_cfg,
+		struct dsc_optc_config *dsc_optc_cfg,
+		bool enable)
+{
+	struct dc *dc = pipe_ctx->stream->ctx->dc;
+	struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
+	struct dc_stream_state *stream = pipe_ctx->stream;
+	enum optc_dsc_mode optc_dsc_mode;
+	int odm_count = 1;
+	struct pipe_ctx *odm_pipe;
+	DC_LOGGER_INIT(dsc->ctx->logger);
 
-/* The stream with these settings can be sent (unblanked) only after DSC was enabled on RX first,
- * i.e. after dp_enable_dsc_on_rx() had been called
+	for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
+		odm_count++;
+
+	if (enable) {
+		optc_dsc_mode = dsc_optc_cfg->is_pixel_format_444 ?
+				OPTC_DSC_ENABLED_444 : OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED;
+
+		if (dc_is_dp_signal(stream->signal) && !dp_is_128b_132b_signal(pipe_ctx)) {
+			DC_LOG_DSC("Setting stream encoder DSC config for engine %d:", (int)pipe_ctx->stream_res.stream_enc->id);
+			dsc_optc_config_log(dsc, dsc_optc_cfg);
+			if (pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config)
+				pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config(pipe_ctx->stream_res.stream_enc,
+										optc_dsc_mode,
+										dsc_optc_cfg->bytes_per_pixel,
+										dsc_optc_cfg->slice_width);
+
+			/* PPS SDP is set elsewhere because it has to be done after DIG FE is connected to DIG BE */
+		} else if (dc_is_hdmi_frl_signal(stream->signal)) {
+			uint8_t dsc_packed_pps[128];
+			struct dc_crtc_timing patched_crtc_timing = stream->timing;
+
+			DC_LOG_DSC("Setting stream encoder DSC config for engine %d:", (int)pipe_ctx->stream_res.hpo_frl_stream_enc->id);
+			dsc_optc_config_log(dsc, dsc_optc_cfg);
+			dsc_cfg->dc_dsc_cfg.num_slices_h *= odm_count;
+			dsc_cfg->pic_width *= odm_count;
+			dsc_cfg->dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding;
+			if (dc->debug.enable_hblank_borrow)
+				dsc_cfg->pic_width = stream->timing.h_addressable;
+			dsc->funcs->dsc_get_packed_pps(dsc, dsc_cfg, &dsc_packed_pps[0]);
+			pipe_ctx->stream_res.hpo_frl_stream_enc->funcs->hdmi_frl_set_dsc_config(
+					pipe_ctx->stream_res.hpo_frl_stream_enc, &patched_crtc_timing, &dsc_packed_pps[0]);
+		}
+
+		return;
+	}
+
+	if (dc_is_dp_signal(stream->signal)) {
+		if (dp_is_128b_132b_signal(pipe_ctx))
+			pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_set_dsc_pps_info_packet(
+										pipe_ctx->stream_res.hpo_dp_stream_enc,
+										false,
+										NULL,
+										true);
+		else {
+			if (pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config)
+				pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config(
+							pipe_ctx->stream_res.stream_enc,
+							OPTC_DSC_DISABLED, 0, 0);
+			pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_pps_info_packet(
+							pipe_ctx->stream_res.stream_enc, false, NULL, true);
+		}
+	} else if (dc_is_hdmi_frl_signal(stream->signal)) {
+		pipe_ctx->stream_res.hpo_frl_stream_enc->funcs->hdmi_frl_set_dsc_config(
+				pipe_ctx->stream_res.hpo_frl_stream_enc, &stream->timing, NULL);
+	}
+}
+
+/* Legacy implementation of DSC configuration sequence. DSC HW block programming
+ * is independent of stream encoder and link configuration operations, and should
+ * be configured separately during front-end programming, not within link DPMS sequences.
  */
-void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
+static bool configure_dsc_hw_block(struct pipe_ctx *pipe_ctx, bool enable,
+		struct dsc_config *dsc_cfg, struct dsc_optc_config *dsc_optc_cfg)
 {
-	/* TODO: Move this to HWSS as this is hardware programming sequence not a
-	 * link layer sequence
-	 */
 	struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
 	struct dc *dc = pipe_ctx->stream->ctx->dc;
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	struct pipe_ctx *odm_pipe;
-	int opp_cnt = 1;
 	struct dccg *dccg = dc->res_pool->dccg;
+	int opp_cnt = 1;
+	DC_LOGGER_INIT(dsc->ctx->logger);
+
 	/* It has been found that when DSCCLK is lower than 16Mhz, we will get DCN
 	 * register access hung. When DSCCLk is based on refclk, DSCCLk is always a
 	 * fixed value higher than 16Mhz so the issue doesn't occur. When DSCCLK is
@@ -741,35 +814,34 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 	 * based on fixed value refclk when timing is smaller than 3x16Mhz (i.e
 	 * 48Mhz) pixel clock to avoid hitting this problem.
 	 */
-	bool should_use_dto_dscclk = (dccg->funcs->set_dto_dscclk != NULL) &&
+	bool should_use_dto_dscclk = dccg->funcs->set_dto_dscclk &&
 			stream->timing.pix_clk_100hz > 480000;
-	DC_LOGGER_INIT(dsc->ctx->logger);
 
 	for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
 		opp_cnt++;
 
 	if (enable) {
-		struct dsc_config dsc_cfg;
-		struct dsc_optc_config dsc_optc_cfg = {0};
 		enum optc_dsc_mode optc_dsc_mode;
+		memset(dsc_cfg, 0, sizeof(*dsc_cfg));
+		memset(dsc_optc_cfg, 0, sizeof(*dsc_optc_cfg));
 
 		/* Enable DSC hw block */
-		dsc_cfg.pic_width = (stream->timing.h_addressable + pipe_ctx->dsc_padding_params.dsc_hactive_padding +
+		dsc_cfg->pic_width = (stream->timing.h_addressable + pipe_ctx->dsc_padding_params.dsc_hactive_padding +
 				stream->timing.h_border_left + stream->timing.h_border_right) / opp_cnt;
-		dsc_cfg.pic_height = stream->timing.v_addressable + stream->timing.v_border_top + stream->timing.v_border_bottom;
-		dsc_cfg.pixel_encoding = stream->timing.pixel_encoding;
-		dsc_cfg.color_depth = stream->timing.display_color_depth;
-		dsc_cfg.is_odm = pipe_ctx->next_odm_pipe ? true : false;
-		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
-		ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0);
-		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
-		dsc_cfg.dsc_padding = 0;
+		dsc_cfg->pic_height = stream->timing.v_addressable + stream->timing.v_border_top + stream->timing.v_border_bottom;
+		dsc_cfg->pixel_encoding = stream->timing.pixel_encoding;
+		dsc_cfg->color_depth = stream->timing.display_color_depth;
+		dsc_cfg->is_odm = pipe_ctx->next_odm_pipe != NULL;
+		dsc_cfg->dc_dsc_cfg = stream->timing.dsc_cfg;
+		ASSERT(dsc_cfg->dc_dsc_cfg.num_slices_h % opp_cnt == 0);
+		dsc_cfg->dc_dsc_cfg.num_slices_h /= opp_cnt;
+		dsc_cfg->dsc_padding = 0;
 
 		if (should_use_dto_dscclk)
-			dccg->funcs->set_dto_dscclk(dccg, dsc->inst, dsc_cfg.dc_dsc_cfg.num_slices_h);
-		if (!dsc->funcs->dsc_prepare_config(dsc, &dsc_cfg, &dsc_optc_cfg)) {
+			dccg->funcs->set_dto_dscclk(dccg, dsc->inst, dsc_cfg->dc_dsc_cfg.num_slices_h);
+		if (!dsc->funcs->dsc_prepare_config(dsc, dsc_cfg, dsc_optc_cfg)) {
 			ASSERT(false);
-			return;
+			return false;
 		}
 		dsc->funcs->dsc_set_config(dsc);
 		dsc->funcs->dsc_enable(dsc, pipe_ctx->stream_res.opp->inst);
@@ -778,112 +850,92 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 			ASSERT(odm_dsc);
 			if (!odm_dsc)
-				return;
+				return false;
 			if (should_use_dto_dscclk)
-				dccg->funcs->set_dto_dscclk(dccg, odm_dsc->inst, dsc_cfg.dc_dsc_cfg.num_slices_h);
-			if (!odm_dsc->funcs->dsc_prepare_config(odm_dsc, &dsc_cfg, &dsc_optc_cfg)) {
+				dccg->funcs->set_dto_dscclk(dccg, odm_dsc->inst,
+						dsc_cfg->dc_dsc_cfg.num_slices_h);
+			if (!odm_dsc->funcs->dsc_prepare_config(odm_dsc, dsc_cfg, dsc_optc_cfg)) {
 				ASSERT(false);
-				return;
+				return false;
 			}
 			odm_dsc->funcs->dsc_set_config(odm_dsc);
 			odm_dsc->funcs->dsc_enable(odm_dsc, odm_pipe->stream_res.opp->inst);
 		}
-		dsc_cfg.dc_dsc_cfg.num_slices_h *= opp_cnt;
-		dsc_cfg.pic_width *= opp_cnt;
-		dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding;
+		optc_dsc_mode = dsc_optc_cfg->is_pixel_format_444 ? OPTC_DSC_ENABLED_444 : OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED;
 
-		optc_dsc_mode = dsc_optc_cfg.is_pixel_format_444 ? OPTC_DSC_ENABLED_444 : OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED;
+		/* Enable DSC in OPTC */
+		DC_LOG_DSC("Setting optc DSC config for tg instance %d:", pipe_ctx->stream_res.tg->inst);
+		dsc_optc_config_log(dsc, dsc_optc_cfg);
+		pipe_ctx->stream_res.tg->funcs->set_dsc_config(pipe_ctx->stream_res.tg,
+							optc_dsc_mode,
+							dsc_optc_cfg->bytes_per_pixel,
+							dsc_optc_cfg->slice_width);
+		return true;
+	}
 
-		/* Enable DSC in encoder */
-		if (dc_is_dp_signal(stream->signal) && !dp_is_128b_132b_signal(pipe_ctx)) {
-			DC_LOG_DSC("Setting stream encoder DSC config for engine %d:", (int)pipe_ctx->stream_res.stream_enc->id);
-			dsc_optc_config_log(dsc, &dsc_optc_cfg);
-			if (pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config)
-				pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config(pipe_ctx->stream_res.stream_enc,
-										optc_dsc_mode,
-										dsc_optc_cfg.bytes_per_pixel,
-										dsc_optc_cfg.slice_width);
+	/* disable DSC in OPTC */
+	pipe_ctx->stream_res.tg->funcs->set_dsc_config(
+			pipe_ctx->stream_res.tg,
+			OPTC_DSC_DISABLED, 0, 0);
 
-			/* PPS SDP is set elsewhere because it has to be done after DIG FE is connected to DIG BE */
-		}
-		else if (dc_is_hdmi_frl_signal(stream->signal)) {
-			uint8_t dsc_packed_pps[128];
-			struct dc_crtc_timing patched_crtc_timing = stream->timing;
+	for (odm_pipe = pipe_ctx; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) {
+		odm_pipe->stream_res.dsc->funcs->dsc_disconnect(odm_pipe->stream_res.dsc);
 
-			DC_LOG_DSC("Setting stream encoder DSC config for engine %d:", (int)pipe_ctx->stream_res.hpo_frl_stream_enc->id);
-			dsc_optc_config_log(dsc, &dsc_optc_cfg);
+		/*
+		 * dsc_disconnect is a double buffered register.
+		 * by the time we call dsc_disable, dsc may still remain
+		 * connected to OPP. In this case OPTC will no longer
+		 * get correct pixel data because DSCC is off. However
+		 * we also can't wait for the  disconnect pending
+		 * complete, because this function can be called
+		 * with/without OTG master lock acquired. When the lock
+		 * is acquired we will never get pending complete until
+		 * we release the lock later. So there is no easy way to
+		 * solve this problem especially when the lock is
+		 * acquired. DSC is a front end hw block it should be
+		 * programmed as part of front end sequence, where the
+		 * commit sequence without lock and update sequence
+		 * with lock are completely separated. However because
+		 * we are programming dsc as part of back end link
+		 * programming sequence, we don't know if front end OPTC
+		 * master lock is acquired. The back end should be
+		 * agnostic to front end lock. DSC programming shouldn't
+		 * belong to this sequence.
+		 */
+		odm_pipe->stream_res.dsc->funcs->dsc_disable(odm_pipe->stream_res.dsc);
+		if (dccg->funcs->set_ref_dscclk)
+			dccg->funcs->set_ref_dscclk(dccg, odm_pipe->stream_res.dsc->inst);
+	}
+	return true;
+}
 
-			/* if we are borrowing from hblank, h_addressable and  pic_width need to be adjusted */
-			if (dc->debug.enable_hblank_borrow) {
-				dsc_cfg.pic_width = stream->timing.h_addressable;
-			}
+/* The stream with these settings can be sent (unblanked) only after DSC was enabled on RX first,
+ * i.e. after dp_enable_dsc_on_rx() had been called
+ */
+void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
+{
+	struct dc *dc = pipe_ctx->stream->ctx->dc;
+	struct dsc_config dsc_cfg = {0};
+	struct dsc_optc_config dsc_optc_cfg = {0};
 
-			dsc->funcs->dsc_get_packed_pps(dsc, &dsc_cfg, &dsc_packed_pps[0]);
-			pipe_ctx->stream_res.hpo_frl_stream_enc->funcs->hdmi_frl_set_dsc_config(
-					pipe_ctx->stream_res.hpo_frl_stream_enc, &patched_crtc_timing, &dsc_packed_pps[0]);
-		}
+	if (!pipe_ctx->stream_res.dsc) {
+		ASSERT(false);
+		return;
+	}
 
-		/* Enable DSC in OPTC */
-		DC_LOG_DSC("Setting optc DSC config for tg instance %d:", pipe_ctx->stream_res.tg->inst);
-		dsc_optc_config_log(dsc, &dsc_optc_cfg);
-		pipe_ctx->stream_res.tg->funcs->set_dsc_config(pipe_ctx->stream_res.tg,
-							optc_dsc_mode,
-							dsc_optc_cfg.bytes_per_pixel,
-							dsc_optc_cfg.slice_width);
+	if (dc->hwss.configure_dsc) {
+		/* If the hwss configure_dsc function is supported, then DSC will be programmed
+		 * in the same sequence of OTG enable/disable. Thus, we don't need to configure
+		 * the DSC hw block here.
+		 */
+		set_encoder_dsc_config(pipe_ctx, &dsc_cfg, &dsc_optc_cfg, false);
 	} else {
-		/* disable DSC in OPTC */
-		pipe_ctx->stream_res.tg->funcs->set_dsc_config(
-				pipe_ctx->stream_res.tg,
-				OPTC_DSC_DISABLED, 0, 0);
-
-		/* disable DSC in stream encoder */
-		if (dc_is_dp_signal(stream->signal)) {
-			if (dp_is_128b_132b_signal(pipe_ctx))
-				pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_set_dsc_pps_info_packet(
-										pipe_ctx->stream_res.hpo_dp_stream_enc,
-										false,
-										NULL,
-										true);
-			else {
-				if (pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config)
-					pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_config(
-							pipe_ctx->stream_res.stream_enc,
-							OPTC_DSC_DISABLED, 0, 0);
-				pipe_ctx->stream_res.stream_enc->funcs->dp_set_dsc_pps_info_packet(
-							pipe_ctx->stream_res.stream_enc, false, NULL, true);
-			}
-		}
-		else if (dc_is_hdmi_frl_signal(stream->signal))
-			pipe_ctx->stream_res.hpo_frl_stream_enc->funcs->hdmi_frl_set_dsc_config(
-					pipe_ctx->stream_res.hpo_frl_stream_enc, &stream->timing, NULL);
-
-		/* disable DSC block */
-		for (odm_pipe = pipe_ctx; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) {
-			odm_pipe->stream_res.dsc->funcs->dsc_disconnect(odm_pipe->stream_res.dsc);
-			/*
-			 * TODO - dsc_disconnect is a double buffered register.
-			 * by the time we call dsc_disable, dsc may still remain
-			 * connected to OPP. In this case OPTC will no longer
-			 * get correct pixel data because DSCC is off. However
-			 * we also can't wait for the  disconnect pending
-			 * complete, because this function can be called
-			 * with/without OTG master lock acquired. When the lock
-			 * is acquired we will never get pending complete until
-			 * we release the lock later. So there is no easy way to
-			 * solve this problem especially when the lock is
-			 * acquired. DSC is a front end hw block it should be
-			 * programmed as part of front end sequence, where the
-			 * commit sequence without lock and update sequence
-			 * with lock are completely separated. However because
-			 * we are programming dsc as part of back end link
-			 * programming sequence, we don't know if front end OPTC
-			 * master lock is acquired. The back end should be
-			 * agnostic to front end lock. DSC programming shouldn't
-			 * belong to this sequence.
-			 */
-			odm_pipe->stream_res.dsc->funcs->dsc_disable(odm_pipe->stream_res.dsc);
-			if (dccg->funcs->set_ref_dscclk)
-				dccg->funcs->set_ref_dscclk(dccg, odm_pipe->stream_res.dsc->inst);
+		if (enable) {
+			if (configure_dsc_hw_block(pipe_ctx, true, &dsc_cfg, &dsc_optc_cfg))
+				set_encoder_dsc_config(pipe_ctx, &dsc_cfg, &dsc_optc_cfg, true);
+		} else {
+			set_encoder_dsc_config(pipe_ctx, &dsc_cfg, &dsc_optc_cfg, false);
+			configure_dsc_hw_block(pipe_ctx, false, &dsc_cfg, &dsc_optc_cfg);
 		}
 	}
 }
@@ -2038,6 +2090,8 @@ static enum dc_status enable_link_hdmi_frl(struct pipe_ctx *pipe_ctx)
 		pipe_ctx->stream_res.hpo_frl_stream_enc->funcs->hdmi_frl_enable(
 			pipe_ctx->stream_res.hpo_frl_stream_enc,
 			pipe_ctx->stream_res.tg->inst);
+	} else if (link_stat == LINK_RESULT_NO_TIMEOUT) {
+		return status;
 	} else {
 		status = DC_FAIL_HDMI_FRL_LINK_TRAINING;
 		stream->link->frl_link_settings.frl_link_rate = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c
index 5e0979ac3dcf..960f18fe275b 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c
@@ -442,7 +442,7 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 	union hdmi_scdc_status_flags_data status_data = {0};
 	union hdmi_scdc_source_test_req test_req = {0};
 	union hdmi_scdc_LTP_req_data ltp_req = {0};
-	unsigned long long poll_timeout_ns = 200 * 1000 * 1000;
+	unsigned long long poll_timeout_ns = 220 * 1000 * 1000;
 	unsigned long long wait_time_ns = 2 * 1000 * 1000;
 	unsigned long long num_polls = 0;
 	struct hpo_frl_link_encoder *hpo_frl_link_enc = ddc_service->link->hpo_frl_link_enc;
@@ -451,11 +451,37 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 	uint8_t FFE_Levels = hdmi_frl_get_max_ffe_level(ddc_service->link, link_settings);
 	uint8_t current_FFE = 0;
 	bool override_FFE = false;
-	bool flt_no_timeout = false;
 
 	DC_LOGGER_INIT(ddc_service->link->ctx->logger);
 	FRL_INFO("FRL LINK TRAINING:  Starting FRL Link Training.\n");
 
+	/*Test Condition - FLT_no_timeout avoid link training*/
+	offset = HDMI_SCDC_SOURCE_TEST_REQ;
+	link_query_ddc_data(ddc_service, slave_address, &offset,
+			sizeof(offset), &test_req.byte, sizeof(test_req.byte));
+	FRL_INFO("FRL TEST REQ:  FLT_no_timeout = %d \n", test_req.fields.FLT_NO_TIMEOUT);
+
+	if (test_req.fields.FLT_NO_TIMEOUT && ddc_service->link->frl_flags.no_timeout) {
+		bool de_emphasis_only = false;
+		bool pre_shoot_only = false;
+		bool no_ffe = false;
+
+		read_idcc_data(ddc_service, HDMI_IDCC_SCOPE_RW_CA,
+			ddc_service->link->hdmi_cable_id.raw, 0,
+			sizeof(ddc_service->link->hdmi_cable_id.raw));
+
+		hdmi_return_preeshoot_and_deemphasis(ddc_service->link, &test_req,
+			&de_emphasis_only, &pre_shoot_only, &no_ffe);
+
+		dio_link_enc->funcs->prog_eq_setting(dio_link_enc, 0,
+			de_emphasis_only,
+			pre_shoot_only,
+			no_ffe,
+			link_settings);
+
+		return LINK_RESULT_NO_TIMEOUT;
+	}
+
 	hdmi_frl_LTS_clear_Link_Setting(ddc_service);
 
 	offset = HDMI_SCDC_SINK_VERSION;
@@ -511,13 +537,68 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 		}
 	} while (elapsed < poll_timeout_ns);
 
-	/*Test Condition - FLT_no_timeout avoid link training*/
-	offset = HDMI_SCDC_SOURCE_TEST_REQ;
-	link_query_ddc_data(ddc_service, slave_address, &offset,
-			sizeof(offset), &test_req.byte, sizeof(test_req.byte));
-	FRL_INFO("FRL TEST REQ:  FLT_no_timeout = %d \n", test_req.fields.FLT_NO_TIMEOUT);
-	if (test_req.fields.FLT_NO_TIMEOUT)
-		flt_no_timeout = true;
+	if (test_req.fields.FLT_NO_TIMEOUT) {
+		ddc_service->link->frl_flags.no_timeout = true;
+
+		/*Specify FRL rate*/
+		write_buffer[0] = HDMI_SCDC_CONFIG_1;
+		/*FRL_RATE*/
+		write_buffer[1] = (uint8_t)(link_settings->frl_link_rate | (FFE_Levels << 4));
+		link_query_ddc_data(ddc_service, slave_address,
+				write_buffer, sizeof(write_buffer), NULL, 0);
+		FRL_INFO("FRL LINK TRAINING:  Write link rate = %d.  Max FFE_Levels = %d\n",
+				link_settings->frl_link_rate, FFE_Levels);
+
+		offset = HDMI_SCDC_UPDATE_0;
+		/*Check FLT_update flag*/
+		link_query_ddc_data(ddc_service, slave_address,
+						&offset, sizeof(offset), &scdc_update.byte[0],
+						sizeof(scdc_update.byte[0]));
+
+		offset = HDMI_SCDC_LTP_REQ;
+		link_query_ddc_data(ddc_service, slave_address,
+						&offset, sizeof(offset), ltp_req.byte,
+						sizeof(ltp_req.byte));
+
+		ln0_pattern = ltp_req.fields.LN0_LTP_REQ;
+		ln1_pattern = ltp_req.fields.LN1_LTP_REQ;
+		ln2_pattern = ltp_req.fields.LN2_LTP_REQ;
+		ln3_pattern = ltp_req.fields.LN3_LTP_REQ;
+
+		hpo_frl_link_enc->funcs->set_hdmi_training_pattern(
+					hpo_frl_link_enc,
+					ln0_pattern - 1,
+					ln1_pattern - 1,
+					ln2_pattern - 1,
+					ln3_pattern - 1);
+
+		read_idcc_data(ddc_service, HDMI_IDCC_SCOPE_RW_CA,
+				ddc_service->link->hdmi_cable_id.raw, 0,
+				sizeof(ddc_service->link->hdmi_cable_id.raw));
+
+		bool de_emphasis_only = false;
+		bool pre_shoot_only = false;
+		bool no_ffe = false;
+
+		hdmi_return_preeshoot_and_deemphasis(ddc_service->link, &test_req,
+			&de_emphasis_only, &pre_shoot_only, &no_ffe);
+
+		dio_link_enc->funcs->prog_eq_setting(dio_link_enc, 0,
+			de_emphasis_only,
+			pre_shoot_only,
+			no_ffe,
+			link_settings);
+
+		write_buffer[0] = HDMI_SCDC_UPDATE_0;
+		/*FLT_update - bit 5*/
+		write_buffer[1] = (scdc_update.fields.FLT_UPDATE << 5);
+		link_query_ddc_data(ddc_service, slave_address,
+				write_buffer, sizeof(write_buffer), NULL, 0);
+
+		return LINK_RESULT_NO_TIMEOUT;
+	}
+
+	ddc_service->link->frl_flags.no_timeout = false;
 
 	if (status_data.fields.FLT_READY) {
 		/*Specify FRL rate*/
@@ -534,9 +615,7 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 		/*Start FLT Timer = 200 ms, or 300ms if link rate >= 16Gbps*/
 		num_polls = 0;
 		if (link_settings->frl_link_rate >= HDMI_FRL_LINK_RATE_16GBPS)
-			poll_timeout_ns = 300 * 1000 * 1000;
-		if (flt_no_timeout)
-			poll_timeout_ns = 1000 * 1000 * 1000;
+			poll_timeout_ns = 320 * 1000 * 1000;
 
 		flt_poll_start = dm_get_timestamp(ddc_service->ctx);
 		uint8_t prev_flt_update = 0;
@@ -561,21 +640,9 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 				prev_flt_update = scdc_update.fields.FLT_UPDATE;
 			}
 
-			if (flt_no_timeout) {
-				offset = HDMI_SCDC_SOURCE_TEST_REQ;
-				link_query_ddc_data(ddc_service, slave_address, &offset,
-						sizeof(offset), &test_req.byte, sizeof(test_req.byte));
-				if (test_req.fields.FLT_NO_TIMEOUT == 0) {
-					FRL_INFO("FRL TEST REQ:  FLT_no_timeout = %d \n", test_req.fields.FLT_NO_TIMEOUT);
-					return LINK_RESULT_SUCCESS;
-				}
-			}
-
 			/*Set TxFFE = TxFFE0*/
 			/*Program FFE_Levels - scdc_config has this field at 0 */
 			if (override_FFE) {
-				if (flt_no_timeout)
-					current_FFE = 0;
 				if (current_FFE == 0)
 					dio_link_enc->funcs->prog_eq_setting(dio_link_enc, current_FFE,
 							test_req.fields.TXFFE_DEEMPHASIS,
@@ -626,8 +693,7 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 
 				if (ln0_pattern == 0x03 || ln1_pattern == 0x03 ||
 						ln2_pattern == 0x03 || ln3_pattern == 0x03)
-					if (!flt_no_timeout)
-						continue;
+					continue;
 
 				if (link_settings->frl_num_lanes == 3) {
 					ln3_pattern = 1;
@@ -651,8 +717,6 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 						override_FFE = true;
 						if (current_FFE > FFE_Levels)
 							current_FFE = 0;
-						if (flt_no_timeout)
-							current_FFE = 0;
 					}
 				} else {
 					if (!ln0_pattern && !ln1_pattern && !ln2_pattern && !ln3_pattern) {
@@ -677,8 +741,6 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 						override_FFE = true;
 						if (current_FFE > FFE_Levels)
 							current_FFE = 0;
-						if (flt_no_timeout)
-							current_FFE = 0;
 					}
 				}
 
@@ -733,7 +795,7 @@ enum link_result hdmi_frl_perform_link_training_with_retries(
 
 	/* FRL Link Training */
 	while (status != LINK_RESULT_FALLBACK) {
-		if (status == LINK_RESULT_SUCCESS) {
+		if (status == LINK_RESULT_SUCCESS || status == LINK_RESULT_NO_TIMEOUT) {
 			break;
 		} else if (status == LINK_RESULT_LOWER_LINKRATE) {
 			FRL_INFO("FRL LINK TRAINING:  Sink requested lower link rate during link enable. \n");
@@ -780,6 +842,9 @@ enum link_result hdmi_frl_perform_link_training_with_fallback(
 		status = hdmi_frl_perform_link_training(link->ddc,
 				&link->frl_link_settings);
 
+		if (status == LINK_RESULT_NO_TIMEOUT)
+			break;
+
 		link->dc->hwss.disable_link_output(link, link_res, SIGNAL_TYPE_HDMI_FRL);
 
 		if (status == LINK_RESULT_SUCCESS)
@@ -892,7 +957,7 @@ void hdmi_frl_verify_link_cap(struct dc_link *link,
 
 	status = hdmi_frl_perform_link_training_with_fallback(link, &link_res, frl_phy_clock_source_id);
 
-	if (status == LINK_RESULT_SUCCESS) {
+	if (status == LINK_RESULT_SUCCESS || status == LINK_RESULT_NO_TIMEOUT) {
 		cur->frl_link_rate = link->frl_link_settings.frl_link_rate;
 		cur->frl_num_lanes = link->frl_link_settings.frl_num_lanes;
 		success = true;
diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h
index 27d4a96b958f..339372293a98 100644
--- a/drivers/gpu/drm/amd/display/dc/os_types.h
+++ b/drivers/gpu/drm/amd/display/dc/os_types.h
@@ -27,7 +27,6 @@
 #ifndef _OS_TYPES_H_
 #define _OS_TYPES_H_
 
-#include <linux/compiler_attributes.h>
 #include <linux/slab.h>
 #include <linux/kgdb.h>
 #include <linux/delay.h>
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
index e5e94e955157..e172163942b4 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
@@ -745,8 +745,8 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.clock_trace = true,
 	.disable_pplib_clock_request = false,
 	.ignore_pg = false,
-	.disable_dpp_power_gate = false,
-	.disable_hubp_power_gate = false,
+	.disable_dpp_power_gate = true,
+	.disable_hubp_power_gate = true,
 	.disable_optc_power_gate = true,
 	.disable_dsc_power_gate = false,
 	.disable_dio_power_gate = true,
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.h
index b1a802f9f4b4..34cfdc6eb8e5 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.h
@@ -370,6 +370,12 @@ bool dcn50_program_mcache_pipe_config(struct dc_state *context,
 	SRI_ARR(CM_TEST_DEBUG_DATA, CM, id),                                     \
 	SRI_ARR(FORMAT_CONTROL, CNVC_CFG, id),                                   \
 	SRI_ARR(CNVC_SURFACE_PIXEL_FORMAT, CNVC_CFG, id),                        \
+	SRI_ARR(CNVC_UPSP_CLAMP, CNVC_CFG, id),                                  \
+	SRI_ARR(CNVC_UPSP_MODE, CNVC_CFG, id),                                   \
+	SRI_ARR(CNVC_UPSP_H_COEF_P0, CNVC_CFG, id),                              \
+	SRI_ARR(CNVC_UPSP_H_COEF_P1, CNVC_CFG, id),                              \
+	SRI_ARR(CNVC_UPSP_V_COEF_P0, CNVC_CFG, id),                              \
+	SRI_ARR(CNVC_UPSP_V_COEF_P1, CNVC_CFG, id),                              \
 	SRI_ARR(CURSOR0_CONTROL, CM_CUR, id),                                    \
 	SRI_ARR(CURSOR0_COLOR0, CM_CUR, id),                                     \
 	SRI_ARR(CURSOR0_COLOR1, CM_CUR, id),                                     \
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
index f14e3069acfd..1eaf039e2023 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
@@ -487,15 +487,17 @@ static void spl_calculate_scaling_ratios(struct spl_in *spl_in,
 	spl_scratch->scl_data.ratios.horz_c = spl_scratch->scl_data.ratios.horz;
 	spl_scratch->scl_data.ratios.vert_c = spl_scratch->scl_data.ratios.vert;
 
-	if (spl_is_yuv420(spl_in->basic_in.format)) {
-		spl_scratch->scl_data.ratios.horz_c.value /= 2;
-		spl_scratch->scl_data.ratios.vert_c.value /= 2;
-	} else if (spl_is_yuv422(spl_in->basic_in.format)) {
-		if (spl_in->basic_in.rotation == SPL_ROTATION_ANGLE_90 ||
-			spl_in->basic_in.rotation == SPL_ROTATION_ANGLE_270)
-			spl_scratch->scl_data.ratios.vert_c.value /= 2;
-		else
+	if (!spl_in->upsp_mode) {
+		if (spl_is_yuv420(spl_in->basic_in.format)) {
 			spl_scratch->scl_data.ratios.horz_c.value /= 2;
+			spl_scratch->scl_data.ratios.vert_c.value /= 2;
+		} else if (spl_is_yuv422(spl_in->basic_in.format)) {
+			if (spl_in->basic_in.rotation == SPL_ROTATION_ANGLE_90 ||
+				spl_in->basic_in.rotation == SPL_ROTATION_ANGLE_270)
+				spl_scratch->scl_data.ratios.vert_c.value /= 2;
+			else
+				spl_scratch->scl_data.ratios.horz_c.value /= 2;
+		}
 	}
 	spl_scratch->scl_data.ratios.horz = spl_fixpt_truncate(
 			spl_scratch->scl_data.ratios.horz, 19);
@@ -669,7 +671,8 @@ static void spl_calculate_inits_and_viewports(struct spl_in *spl_in,
 			&flip_vert_scan_dir,
 			&flip_horz_scan_dir);
 
-	if (spl_is_subsampled_format(spl_in->basic_in.format)) {
+	if (spl_is_subsampled_format(spl_in->basic_in.format) &&
+			!spl_in->upsp_mode) {
 		/* this gives the direction of the cositing (negative will move
 		 * left, right otherwise)
 		 */
@@ -804,6 +807,9 @@ static enum scl_mode spl_get_dscl_mode(const struct spl_in *spl_in,
 			&& !enable_isharp)
 		return SCL_MODE_SCALING_444_BYPASS;
 
+	if (spl_in->upsp_mode)
+		return SCL_MODE_SCALING_444_RGB_ENABLE;
+
 	if (!spl_is_subsampled_format(pixel_format)) {
 		if (spl_is_video_format(pixel_format))
 			return SCL_MODE_SCALING_444_YCBCR_ENABLE;
@@ -824,9 +830,9 @@ static enum scl_mode spl_get_dscl_mode(const struct spl_in *spl_in,
 }
 
 static void spl_choose_lls_policy(enum spl_pixel_format format,
-	enum linear_light_scaling *lls_pref)
+	enum linear_light_scaling *lls_pref, enum upsp_mode upsp_mode)
 {
-	if (spl_is_subsampled_format(format))
+	if (spl_is_subsampled_format(format) && !upsp_mode)
 		*lls_pref = LLS_PREF_NO;
 	else /* RGB or YUV444 */
 		*lls_pref = LLS_PREF_YES;
@@ -858,7 +864,7 @@ static bool enable_easf(struct spl_in *spl_in, struct spl_scratch *spl_scratch)
 	 */
 	if (spl_in->lls_pref == LLS_PREF_DONT_CARE)
 		spl_choose_lls_policy(spl_in->basic_in.format,
-			&spl_in->lls_pref);
+			&spl_in->lls_pref, spl_in->upsp_mode);
 
 	/* Check for linear scaling or EASF preferred */
 	if (spl_in->lls_pref != LLS_PREF_YES && !spl_in->prefer_easf)
@@ -1009,8 +1015,15 @@ static bool spl_get_optimal_number_of_taps(
 	unsigned int min_taps_y, min_taps_c;
 	enum lb_memory_config lb_config;
 	bool skip_easf          = false;
-	bool is_horz_subsampled = spl_is_subsampled_format(spl_in->basic_in.format);
-	bool is_vert_subsampled = spl_is_yuv420(spl_in->basic_in.format);
+	bool is_horz_subsampled = (spl_is_subsampled_format(spl_in->basic_in.format) && !spl_in->upsp_mode);
+	bool is_vert_subsampled = (spl_is_yuv420(spl_in->basic_in.format) && !spl_in->upsp_mode);
+
+	/* Only 4:2:2 subsamples a single axis; 90/270 rotates that axis from horizontal to vertical
+	 * (like the ratio/cositing swaps above) so the now-full-res axis gets the 1-tap sharp bypass */
+	if ((spl_in->basic_in.rotation == SPL_ROTATION_ANGLE_90 ||
+		spl_in->basic_in.rotation == SPL_ROTATION_ANGLE_270) &&
+		spl_is_yuv422(spl_in->basic_in.format))
+		spl_swap(is_horz_subsampled, is_vert_subsampled);
 
 	if (spl_scratch->scl_data.viewport.width > spl_scratch->scl_data.h_active &&
 		max_downscale_src_width != 0 &&
@@ -1043,7 +1056,8 @@ static bool spl_get_optimal_number_of_taps(
 		spl_get_taps_non_adaptive_scaler(spl_scratch, in_taps, is_horz_subsampled, is_vert_subsampled);
 	}
 	else {
-		if (spl_is_subsampled_format(spl_in->basic_in.format)) {
+		if (spl_is_subsampled_format(spl_in->basic_in.format) &&
+				!spl_in->upsp_mode) {
 			spl_scratch->scl_data.taps.h_taps = 6;
 			spl_scratch->scl_data.taps.v_taps = 6;
 			spl_scratch->scl_data.taps.h_taps_c = 4;
@@ -1213,9 +1227,10 @@ static bool spl_get_optimal_number_of_taps(
 }
 
 static void spl_set_black_color_data(enum spl_pixel_format format,
-			struct scl_black_color *scl_black_color)
+		struct scl_black_color *scl_black_color,
+		enum upsp_mode upsp_mode)
 {
-	bool ycbcr = spl_is_video_format(format);
+	bool ycbcr = (spl_is_video_format(format) && !upsp_mode);
 	if (ycbcr)	{
 		scl_black_color->offset_rgb_y = BLACK_OFFSET_RGB_Y;
 		scl_black_color->offset_rgb_cbcr = BLACK_OFFSET_CBCR;
@@ -1298,8 +1313,8 @@ static void spl_set_dscl_prog_data(struct spl_in *spl_in, struct spl_scratch *sp
 		enable_easf);
 
 	// SCL_BLACK_COLOR
-	spl_set_black_color_data(spl_in->basic_in.format, scl_black_color);
-
+	spl_set_black_color_data(spl_in->basic_in.format, scl_black_color,
+			spl_in->upsp_mode);
 	/* Manually calculate scale ratio and init values */
 	spl_set_manual_ratio_init_data(dscl_prog_data, data);
 
@@ -1358,7 +1373,7 @@ static void spl_calculate_c0_c3_hdr(struct dscl_prog_data *dscl_prog_data, uint3
 static void spl_set_easf_data(struct spl_scratch *spl_scratch, struct spl_out *spl_out, bool enable_easf_v,
 	bool enable_easf_h, enum linear_light_scaling lls_pref,
 	enum spl_pixel_format format, enum system_setup setup,
-	uint32_t sdr_white_level_nits)
+	uint32_t sdr_white_level_nits, enum upsp_mode upsp_mode)
 {
 	struct dscl_prog_data *dscl_prog_data = spl_out->dscl_prog_data;
 	if (enable_easf_v) {
@@ -1689,7 +1704,7 @@ static void spl_set_easf_data(struct spl_scratch *spl_scratch, struct spl_out *s
 			0x0;	// fp1.5.10, C3 coefficient
 	}
 
-	if (spl_is_subsampled_format(format)) { /* TODO: 0 = RGB, 1 = YUV */
+	if (spl_is_subsampled_format(format) && !upsp_mode) { /* TODO: 0 = RGB, 1 = YU//V */
 		dscl_prog_data->easf_matrix_mode = 1;
 		/*
 		 * 2-bit, BF3 chroma mode correction calculation mode
@@ -2073,7 +2088,7 @@ bool SPL_NAMESPACE(spl_calculate_scaler_params(struct spl_in *spl_in, struct spl
 
 	// Set EASF
 	spl_set_easf_data(&spl_scratch, spl_out, enable_easf_v, enable_easf_h, spl_in->lls_pref,
-		spl_in->basic_in.format, setup, spl_in->sdr_white_level_nits);
+		spl_in->basic_in.format, setup, spl_in->sdr_white_level_nits, spl_in->upsp_mode);
 
 	// Set iSHARP
 	vratio = spl_fixpt_ceil(spl_scratch.scl_data.ratios.vert);
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index 525e679b81d5..31f147547605 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -1192,6 +1192,10 @@ enum dmub_dc_bls_command_type {
 	 * Placeholder for block-level initialization of the DCHVM
 	 */
 	DMUB_CMD__DC_BLS_DCHVM_INIT = 0,
+	/**
+	 * Runs the fixed DCN hardware init sequence.
+	 */
+	DMUB_CMD__DC_BLS_DCN_INIT = 1,
 };
 
 /**
@@ -2798,8 +2802,11 @@ union dmub_fams2_stream_static_sub_state_v2 {
 
 struct dmub_fams2_stream_static_state {
 	enum fams2_stream_type type;
-	uint32_t otg_vline_time_ns;
-	uint32_t otg_vline_time_ticks;
+	union {
+		uint32_t otg_vline_time_ns; // used by DCN4
+		uint32_t otg_vline_time_ps; // used by DCN6+
+	};
+	uint32_t otg_vline_time_ticks; // U22.10 fixed point, see FAMS2_OTG_VLINE_TIME_TICKS_U22_10_FRAC_BITS
 	uint16_t htotal;
 	uint16_t vtotal; // nominal vtotal
 	uint16_t vblank_start;
@@ -2831,8 +2838,11 @@ struct dmub_fams2_stream_static_state {
 
 struct dmub_fams2_cmd_stream_static_base_state {
 	enum fams2_stream_type type;
-	uint32_t otg_vline_time_ns;
-	uint32_t otg_vline_time_ticks;
+	union {
+		uint32_t otg_vline_time_ns; // used by DCN4
+		uint32_t otg_vline_time_ps; // used by DCN6+
+	};
+	uint32_t otg_vline_time_ticks; // U22.10 fixed point, see FAMS2_OTG_VLINE_TIME_TICKS_U22_10_FRAC_BITS
 	uint16_t htotal;
 	uint16_t vtotal; // nominal vtotal
 	uint16_t vblank_start;
@@ -7622,6 +7632,17 @@ struct dmub_rb_cmd_boot_time_crc_init {
 };
 
 /**
+ * Definition of a DMUB_CMD__DC_BLS_DCN_INIT command.
+ *
+ * Runs a fixed DCN hardware init sequence after the pre-OS to post-OS
+ * transition. Carries no payload; anything driver-configurable is
+ * programmed by the driver after this command completes.
+ */
+struct dmub_rb_cmd_dc_bls_dcn_init {
+	struct dmub_cmd_header header; /**< Command header */
+};
+
+/**
  * union dmub_rb_cmd - DMUB inbox command.
  */
 union dmub_rb_cmd {
@@ -8022,6 +8043,11 @@ union dmub_rb_cmd {
 	struct dmub_rb_cmd_boot_time_crc_init boot_time_crc_init;
 
 	/**
+	 * Definition of a DMUB_CMD__DC_BLS_DCN_INIT command.
+	 */
+	struct dmub_rb_cmd_dc_bls_dcn_init dc_bls_dcn_init;
+
+	/**
 	 * Definition of a DMUB_CMD__PANEL_POLARITY_ENABLE command.
 	 */
 	struct dmub_rb_cmd_panel_polarity_enable panel_polarity_enable;
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
index 53e70c682a77..8ebd3d948567 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
@@ -384,10 +384,17 @@ bool dmub_dcn42_is_hw_init(struct dmub_srv *dmub)
 {
 	union dmub_fw_boot_status status;
 	uint32_t is_hw_init;
+	uint32_t fw_version;
+	bool is_vbios_fw;
 
 	status.all = REG_READ(DMCUB_SCRATCH0);
 	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_hw_init);
 
+	fw_version = REG_READ(DMCUB_SCRATCH1);
+	is_vbios_fw = ((fw_version >> 6) & 0x01) ? true : false;
+	if (is_vbios_fw)
+		return false;
+
 	return is_hw_init != 0 && status.bits.dal_fw;
 }
 
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
index 5181d889fe7f..45ba1a974094 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
@@ -52,9 +52,11 @@ enum adaptive_sync_sdp_version {
 	AS_SDP_VER_0 = 0x0,
 	AS_SDP_VER_1 = 0x1,
 	AS_SDP_VER_2 = 0x2,
+	AS_SDP_VER_3 = 0x3,
 };
 
 #define AS_DP_SDP_LENGTH (9)
+#define AS_DP_SDP_V3_LENGTH (0xA)
 
 struct frame_duration_op {
 	bool          support;
@@ -78,6 +80,9 @@ void mod_build_adaptive_sync_infopacket(const struct dc_stream_state *stream,
 void mod_build_adaptive_sync_infopacket_v2(const struct dc_stream_state *stream,
 		const struct AS_Df_params *param, struct dc_info_packet *info_packet);
 
+void mod_build_adaptive_sync_infopacket_v3(const struct dc_stream_state *stream,
+		const struct AS_Df_params *param, struct dc_info_packet *info_packet);
+
 void mod_build_adaptive_sync_infopacket_v1(struct dc_info_packet *info_packet);
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
index 32b697f46788..8dd9cec6aefb 100644
--- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
@@ -669,8 +669,8 @@ static void build_vtem_infopacket_data(const struct dc_stream_state *stream,
 	set_field_with_mask(&infopacket->sb[VTEM_MD0], MASK_VTEM_MD0__FVA_FACTOR_M1,
 			(fva_factor > 0) ? (fva_factor - 1) : 0);
 	/* VRR Parameters */
-	if (vrr->state == VRR_STATE_ACTIVE_VARIABLE ||
-	    vrr->state == VRR_STATE_ACTIVE_FIXED) {
+	if (vrr->state != VRR_STATE_DISABLED &&
+			vrr->state != VRR_STATE_UNSUPPORTED) {
 		set_field_with_mask(&infopacket->sb[VTEM_MD0], MASK_VTEM_MD0__VRR_EN, 1);
 	} else {
 		set_field_with_mask(&infopacket->sb[VTEM_MD0], MASK_VTEM_MD0__VRR_EN, 0);
@@ -770,9 +770,12 @@ void mod_build_adaptive_sync_infopacket(const struct dc_stream_state *stream,
 	case FREESYNC_TYPE_PCON_IN_WHITELIST:
 	case ADAPTIVE_SYNC_TYPE_EDP:
 		if (stream && stream->link->replay_settings.config.replay_supported &&
-			stream->link->replay_settings.config.replay_version == DC_VESA_PANEL_REPLAY)
-			mod_build_adaptive_sync_infopacket_v2(stream, param, info_packet);
-		else
+			stream->link->replay_settings.config.replay_version == DC_VESA_PANEL_REPLAY) {
+			if (stream->link->dpcd_caps.adaptive_sync_caps.dp_adap_sync_caps.bits.AS_SDP_VERSION_3_SUPPORTED)
+				mod_build_adaptive_sync_infopacket_v3(stream, param, info_packet);
+			else
+				mod_build_adaptive_sync_infopacket_v2(stream, param, info_packet);
+		} else
 			mod_build_adaptive_sync_infopacket_v1(info_packet);
 		break;
 	case ADAPTIVE_SYNC_TYPE_NONE:
@@ -815,3 +818,28 @@ void mod_build_adaptive_sync_infopacket_v2(const struct dc_stream_state *stream,
 	}
 }
 
+void mod_build_adaptive_sync_infopacket_v3(const struct dc_stream_state *stream,
+		const struct AS_Df_params *param,
+		struct dc_info_packet *info_packet)
+{
+	info_packet->valid = true;
+	// HEADER {HB0, HB1, HB2, HB3} = {00, Type, Version, Length}
+	// For eDP VESA Panel Replay the coasting vtotal (DB7-DB9) is filled by the
+	// DMUB firmware,so only the header is built.
+	info_packet->hb0 = 0x00;
+	info_packet->hb1 = 0x22;
+	info_packet->hb2 = AS_SDP_VER_3;
+	info_packet->hb3 = AS_DP_SDP_V3_LENGTH;
+
+	if (param) {
+		//Payload
+		info_packet->sb[0] = param->supportMode; //1: AVT; 0: FAVT
+		info_packet->sb[1] = (stream->timing.v_total & 0x00FF);
+		info_packet->sb[2] = (stream->timing.v_total & 0xFF00) >> 8;
+		//info_packet->sb[3] = 0x00; Target RR, not use fot AVT
+		info_packet->sb[4] = (param->increase.support << 6 | param->decrease.support << 7);
+		info_packet->sb[5] = param->increase.frame_duration_hex;
+		info_packet->sb[6] = param->decrease.frame_duration_hex;
+	}
+}
+
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_0_offset.h
index a3bcdf632066..3dddaec23bcf 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_0_offset.h
@@ -4261,6 +4261,10 @@
 #define regCP_ME2_PIPE2_INT_STATUS_BASE_IDX                                                             0
 #define regCP_ME2_PIPE3_INT_STATUS                                                                      0x1e34
 #define regCP_ME2_PIPE3_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_INT_STAT_DEBUG                                                                        0x1e35
+#define regCP_ME1_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define regCP_ME2_INT_STAT_DEBUG                                                                        0x1e36
+#define regCP_ME2_INT_STAT_DEBUG_BASE_IDX                                                               0
 #define regCP_GFX_QUEUE_INDEX                                                                           0x1e37
 #define regCP_GFX_QUEUE_INDEX_BASE_IDX                                                                  0
 #define regCC_GC_EDC_CONFIG                                                                             0x1e38
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_0_sh_mask.h
index 3088a4a13cb5..1b64f157467d 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_0_sh_mask.h
@@ -16187,6 +16187,60 @@
 #define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
 #define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
 #define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+//CP_ME1_INT_STAT_DEBUG
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//CP_ME2_INT_STAT_DEBUG
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
 //CP_GFX_QUEUE_INDEX
 #define CP_GFX_QUEUE_INDEX__QUEUE_ACCESS__SHIFT                                                               0x0
 #define CP_GFX_QUEUE_INDEX__PIPE_ID__SHIFT                                                                    0x4
diff --git a/drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_4_10_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_4_10_0_offset.h
new file mode 100644
index 000000000000..fe0e0d588c06
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_4_10_0_offset.h
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _nbif_4_10_0_OFFSET_HEADER
+#define _nbif_4_10_0_OFFSET_HEADER
+
+#define regBIF_BX_PF0_RSMU_INDEX                                                                        0x0000
+#define regBIF_BX_PF0_RSMU_INDEX_BASE_IDX                                                               1
+#define regBIF_BX_PF0_RSMU_DATA                                                                         0x0001
+#define regBIF_BX_PF0_RSMU_DATA_BASE_IDX                                                                1
+#define regBIF_BX0_PCIE_INDEX                                                                           0x000c
+#define regBIF_BX0_PCIE_INDEX_BASE_IDX                                                                  0
+#define regBIF_BX0_PCIE_DATA                                                                            0x000d
+#define regBIF_BX0_PCIE_DATA_BASE_IDX                                                                   0
+#define regRCC_STRAP0_RCC_BIF_STRAP2                                                                    0x0006
+#define regRCC_STRAP0_RCC_BIF_STRAP2_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP3                                                                    0x0007
+#define regRCC_STRAP0_RCC_BIF_STRAP3_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP5                                                                    0x0009
+#define regRCC_STRAP0_RCC_BIF_STRAP5_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0                                                              0x0021
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     2
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL                                                            0x0054
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   2
+#define regRCC_DEV0_EPF0_RCC_DOORBELL_APER_EN                                                           0x00c0
+#define regRCC_DEV0_EPF0_RCC_DOORBELL_APER_EN_BASE_IDX                                                  2
+#define regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE                                                           0x00c3
+#define regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE_BASE_IDX                                                  2
+#define regBIF_BX0_INTERRUPT_CNTL                                                                       0x00f1
+#define regBIF_BX0_INTERRUPT_CNTL_BASE_IDX                                                              2
+#define regBIF_BX0_INTERRUPT_CNTL2                                                                      0x00f2
+#define regBIF_BX0_INTERRUPT_CNTL2_BASE_IDX                                                             2
+#define regBIF_BX0_BIF_DOORBELL_INT_CNTL                                                                0x00fe
+#define regBIF_BX0_BIF_DOORBELL_INT_CNTL_BASE_IDX                                                       2
+#define regBIF_BX0_BIF_FB_EN                                                                            0x0100
+#define regBIF_BX0_BIF_FB_EN_BASE_IDX                                                                   2
+#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL                                                             0x012d
+#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL_BASE_IDX                                                    2
+#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL                                                             0x012e
+#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL_BASE_IDX                                                    2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                              0x00f3
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                                     2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                               0x00f4
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                                      2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL                                                   0x00f5
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                          2
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL                                                      0x00f7
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                             2
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ                                                                 0x0106
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ_BASE_IDX                                                        2
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE                                                                0x0107
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE_BASE_IDX                                                       2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL                                                           0x4f0aeb
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1                                                          0x4f0aec
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL                                                           0x4f0aed
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1                                                          0x4f0aee
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL                                                           0x4f0aef
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1                                                          0x4f0af0
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL                                                           0x4f0af1
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1                                                          0x4f0af2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL                                                           0x4f0af3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1                                                          0x4f0af4
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL                                                           0x4f0af5
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1                                                          0x4f0af6
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF2_STRAP2                                                                         0xd102
+#define regRCC_DEV0_EPF2_STRAP2_BASE_IDX                                                                5
+#define regREGS_ROM_OFFSET_CTRL                                                                         0xcc23
+#define regREGS_ROM_OFFSET_CTRL_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE                                                           0x2ffc0de3
+#define regRCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE_BASE_IDX                                                  5
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_4_10_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_4_10_0_sh_mask.h
new file mode 100644
index 000000000000..d8164039a4d3
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_4_10_0_sh_mask.h
@@ -0,0 +1,573 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _nbif_4_10_0_SH_MASK_HEADER
+#define _nbif_4_10_0_SH_MASK_HEADER
+
+//BIF_BX_PF0_RSMU_INDEX
+#define BIF_BX_PF0_RSMU_INDEX__RSMU_INDEX__SHIFT                                                              0x0
+#define BIF_BX_PF0_RSMU_INDEX__RSMU_INDEX_MASK                                                                0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_DATA
+#define BIF_BX_PF0_RSMU_DATA__RSMU_DATA__SHIFT                                                                0x0
+#define BIF_BX_PF0_RSMU_DATA__RSMU_DATA_MASK                                                                  0xFFFFFFFFL
+//BIF_BX0_PCIE_INDEX
+#define BIF_BX0_PCIE_INDEX__PCIE_INDEX__SHIFT                                                                 0x0
+#define BIF_BX0_PCIE_INDEX__PCIE_INDEX_MASK                                                                   0xFFFFFFFFL
+//BIF_BX0_PCIE_DATA
+#define BIF_BX0_PCIE_DATA__PCIE_DATA__SHIFT                                                                   0x0
+#define BIF_BX0_PCIE_DATA__PCIE_DATA_MASK                                                                     0xFFFFFFFFL
+//RCC_STRAP0_RCC_BIF_STRAP2
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SPT__SHIFT                                                      0x1
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWDS_SPT__SHIFT                                                      0x2
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN__SHIFT                                          0x7
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS__SHIFT                                               0xc
+#define RCC_STRAP0_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ARI_EN_UP__SHIFT                                                     0x19
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SPT_MASK                                                        0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWDS_SPT_MASK                                                        0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN_MASK                                            0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS_MASK                                                 0x00001000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ARI_EN_UP_MASK                                                       0x02000000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP3
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP0_RCC_BIF_STRAP5
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN__SHIFT                                 0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN_MASK                                   0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                       0x0
+#define RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                         0x00000001L
+//RCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                               0xFFFFFFFFL
+//BIF_BX0_INTERRUPT_CNTL
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT                                                   0x0
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT                                                         0x1
+#define BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT                                                     0x3
+#define BIF_BX0_INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT                                                       0x4
+#define BIF_BX0_INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT                                                          0x8
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN__SHIFT                                                 0xf
+#define BIF_BX0_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN__SHIFT                                               0x10
+#define BIF_BX0_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS__SHIFT                                   0x11
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN__SHIFT                                              0x12
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK                                                     0x00000001L
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK                                                           0x00000002L
+#define BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK                                                       0x00000008L
+#define BIF_BX0_INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK                                                         0x000000F0L
+#define BIF_BX0_INTERRUPT_CNTL__GEN_IH_INT_EN_MASK                                                            0x00000100L
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN_MASK                                                   0x00008000L
+#define BIF_BX0_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN_MASK                                                 0x00010000L
+#define BIF_BX0_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS_MASK                                     0x00020000L
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN_MASK                                                0x00040000L
+//BIF_BX0_INTERRUPT_CNTL2
+#define BIF_BX0_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT                                                      0x0
+#define BIF_BX0_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK                                                        0xFFFFFFFFL
+//BIF_BX0_BIF_DOORBELL_INT_CNTL
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS__SHIFT                                       0x0
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS__SHIFT                                      0x1
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS__SHIFT                            0x2
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR__SHIFT                                        0x10
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR__SHIFT                                       0x11
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR__SHIFT                             0x12
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                            0x17
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE__SHIFT                                      0x18
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE__SHIFT                                     0x19
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE__SHIFT                           0x1a
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                               0x1c
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1d
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1e
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                              0x1f
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS_MASK                                         0x00000001L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS_MASK                                        0x00000002L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS_MASK                              0x00000004L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR_MASK                                          0x00010000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR_MASK                                         0x00020000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR_MASK                               0x00040000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE_MASK                              0x00800000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE_MASK                                        0x01000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE_MASK                                       0x02000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE_MASK                             0x04000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE_MASK                                 0x10000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x20000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x40000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE_MASK                                0x80000000L
+//BIF_BX0_BIF_FB_EN
+#define BIF_BX0_BIF_FB_EN__FB_READ_EN__SHIFT                                                                  0x0
+#define BIF_BX0_BIF_FB_EN__FB_WRITE_EN__SHIFT                                                                 0x1
+#define BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK                                                                    0x00000001L
+#define BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK                                                                   0x00000002L
+//BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL
+#define BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL
+#define BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT          0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK            0xFFFFFFFFL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT            0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK              0xFFFFFFFFL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT                      0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT                    0x1
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT                    0x8
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK                        0x00000001L
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK                      0x00000002L
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK                      0x000FFF00L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                              0x0
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                              0x1
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                              0x2
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                              0x3
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                              0x4
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                              0x5
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                              0x6
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                              0x7
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                              0x8
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                              0x9
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                            0xa
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                            0xb
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                                        0xc
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                                        0xd
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                                        0xe
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                                        0xf
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                                        0x10
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                                        0x11
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                                        0x12
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                                        0x13
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                                        0x14
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                                        0x15
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                                       0x16
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                                       0x17
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                                       0x18
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                                       0x19
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                                       0x1a
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                                       0x1b
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                                       0x1c
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                                       0x1d
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                                       0x1e
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                                       0x1f
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                                0x00000001L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                                0x00000002L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                                0x00000004L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                                0x00000008L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                                0x00000010L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                                0x00000020L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                                0x00000040L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                                0x00000080L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                                0x00000100L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                                0x00000200L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                              0x00000400L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                              0x00000800L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                          0x00001000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                          0x00002000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                          0x00004000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                          0x00008000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                          0x00010000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                          0x00020000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                          0x00040000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                          0x00080000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                          0x00100000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                          0x00200000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                                         0x00400000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                                         0x00800000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                                         0x01000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                                         0x02000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                                         0x04000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                                         0x08000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                                         0x10000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                                         0x20000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                                         0x40000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                                         0x80000000L
+//BIF_BX_PF0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                             0x0
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                             0x1
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                             0x2
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                             0x3
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                             0x4
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                             0x5
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                             0x6
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                             0x7
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                             0x8
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                             0x9
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                           0xa
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                           0xb
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                                       0xc
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                                       0xd
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                                       0xe
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                                       0xf
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                                       0x10
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                                       0x11
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                                       0x12
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                                       0x13
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                                       0x14
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                                       0x15
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                                      0x16
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                                      0x17
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                                      0x18
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                                      0x19
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                                      0x1a
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                                      0x1b
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                                      0x1c
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                                      0x1d
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                                      0x1e
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                                      0x1f
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                               0x00000001L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                               0x00000002L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                               0x00000004L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                               0x00000008L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                               0x00000010L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                               0x00000020L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                               0x00000040L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                               0x00000080L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                               0x00000100L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                               0x00000200L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                             0x00000400L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                             0x00000800L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                                         0x00001000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                                         0x00002000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                                         0x00004000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                                         0x00008000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                                         0x00010000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                                         0x00020000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                                         0x00040000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                                         0x00080000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                                         0x00100000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                                         0x00200000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                                        0x00400000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                                        0x00800000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                                        0x01000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                                        0x02000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                                        0x04000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                                        0x08000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                                        0x10000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                                        0x20000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                                        0x40000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                                        0x80000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//RCC_DEV0_EPF2_STRAP2
+#define RCC_DEV0_EPF2_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F2__SHIFT                               0x4
+#define RCC_DEV0_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F2__SHIFT                                              0x7
+#define RCC_DEV0_EPF2_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F2__SHIFT                                              0x8
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F2__SHIFT                                            0x9
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F2__SHIFT                                     0xe
+#define RCC_DEV0_EPF2_STRAP2__STRAP_AER_EN_DEV0_F2__SHIFT                                                     0x10
+#define RCC_DEV0_EPF2_STRAP2__STRAP_ACS_EN_DEV0_F2__SHIFT                                                     0x11
+#define RCC_DEV0_EPF2_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F2__SHIFT                                           0x14
+#define RCC_DEV0_EPF2_STRAP2__STRAP_DPA_EN_DEV0_F2__SHIFT                                                     0x15
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F2__SHIFT                                              0x18
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_EN_DEV0_F2__SHIFT                                                   0x1c
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F2__SHIFT                             0x1d
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F2__SHIFT                          0x1e
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F2__SHIFT                                  0x1f
+#define RCC_DEV0_EPF2_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F2_MASK                                 0x00000010L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F2_MASK                                                0x00000080L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F2_MASK                                                0x00000100L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F2_MASK                                              0x00003E00L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F2_MASK                                       0x00004000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_AER_EN_DEV0_F2_MASK                                                       0x00010000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_ACS_EN_DEV0_F2_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F2_MASK                                             0x00100000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_DPA_EN_DEV0_F2_MASK                                                       0x00200000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F2_MASK                                                0x07000000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_EN_DEV0_F2_MASK                                                     0x10000000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F2_MASK                               0x20000000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F2_MASK                            0x40000000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F2_MASK                                    0x80000000L
+//REGS_ROM_OFFSET_CTRL
+#define REGS_ROM_OFFSET_CTRL__ROM_OFFSET__SHIFT                                                               0x0
+#define REGS_ROM_OFFSET_CTRL__ROM_OFFSET_MASK                                                                 0x7FL
+//RCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                               0xFFFFFFFFL
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 4b863a2e38f1..7db44fa24042 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -449,6 +449,8 @@ struct amd_pm_funcs {
 	int (*print_clock_levels)(void *handle, enum pp_clock_type type, char *buf);
 	int (*emit_clock_levels)(void *handle, enum pp_clock_type type, char *buf, int *offset);
 	int (*force_performance_level)(void *handle, enum amd_dpm_forced_level level);
+	int (*reapply_performance_level)(void *handle,
+					 enum amd_dpm_forced_level level);
 	int (*get_sclk_od)(void *handle);
 	int (*set_sclk_od)(void *handle, uint32_t value);
 	int (*get_mclk_od)(void *handle);
@@ -530,6 +532,7 @@ struct amd_pm_funcs {
 	ssize_t (*get_temp_metrics)(void *handle, enum smu_temp_metric_type type, void *table);
 	bool (*temp_metrics_is_supported)(void *handle, enum smu_temp_metric_type type);
 	u64 (*get_npm_cap)(void *handle);
+	int (*set_npm_limit)(void *handle, uint32_t limit);
 	ssize_t (*get_xcp_metrics)(void *handle, int xcp_id, void *table);
 	ssize_t (*get_pm_metrics)(void *handle, void *pmmetrics, size_t size);
 	int (*set_watermarks_for_clock_ranges)(void *handle,
diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/gpu/drm/amd/include/mes_v11_api_def.h
index e251ccc41f9c..f8eaca8a22f6 100644
--- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h
+++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h
@@ -637,6 +637,13 @@ struct WRITE_REG {
 struct READ_REG {
 	uint32_t                  reg_offset;
 	uint64_t                  buffer_addr;
+	union {
+		struct {
+			uint32_t read64Bits : 1;
+			uint32_t reserved : 31;
+		} bits;
+		uint32_t all;
+	} option;
 };
 
 enum WRM_OPERATION {
diff --git a/drivers/gpu/drm/amd/include/v12_structs.h b/drivers/gpu/drm/amd/include/v12_structs.h
index 7fa0c07b51c6..425117fbe7f2 100644
--- a/drivers/gpu/drm/amd/include/v12_structs.h
+++ b/drivers/gpu/drm/amd/include/v12_structs.h
@@ -671,6 +671,139 @@ struct v12_sdma_mqd {
 	uint32_t sdma_queue_id;
 };
 
+/* GFX 12.1 SDMA MQD: diverges from v12_sdma_mqd at offset 0x2B, same total size. */
+struct v12_1_sdma_mqd {
+    uint32_t sdmax_queuen_rb_cntl; // offset: 0  (0x0)
+    uint32_t sdmax_queuen_rb_base; // offset: 1  (0x1)
+    uint32_t sdmax_queuen_rb_base_hi; // offset: 2  (0x2)
+    uint32_t sdmax_queuen_rb_rptr; // offset: 3  (0x3)
+    uint32_t sdmax_queuen_rb_rptr_hi; // offset: 4  (0x4)
+    uint32_t sdmax_queuen_rb_wptr; // offset: 5  (0x5)
+    uint32_t sdmax_queuen_rb_wptr_hi; // offset: 6  (0x6)
+    uint32_t sdmax_queuen_rb_rptr_addr_lo; // offset: 7  (0x7)
+    uint32_t sdmax_queuen_rb_rptr_addr_hi; // offset: 8  (0x8)
+    uint32_t sdmax_queuen_ib_cntl; // offset: 9  (0x9)
+    uint32_t sdmax_queuen_ib_rptr; // offset: 10  (0xA)
+    uint32_t sdmax_queuen_ib_offset; // offset: 11  (0xB)
+    uint32_t sdmax_queuen_ib_base_lo; // offset: 12  (0xC)
+    uint32_t sdmax_queuen_ib_base_hi; // offset: 13  (0xD)
+    uint32_t sdmax_queuen_ib_size; // offset: 14  (0xE)
+    uint32_t sdmax_queuen_doorbell; // offset: 15  (0xF)
+    uint32_t sdmax_queuen_doorbell_log; // offset: 16  (0x10)
+    uint32_t sdmax_queuen_doorbell_offset; // offset: 17  (0x11)
+    uint32_t sdmax_queuen_csa_addr_lo; // offset: 18  (0x12)
+    uint32_t sdmax_queuen_csa_addr_hi; // offset: 19  (0x13)
+    uint32_t sdmax_queuen_sched_cntl; // offset: 20  (0x14)
+    uint32_t sdmax_queuen_ib_sub_remain; // offset: 21  (0x15)
+    uint32_t sdmax_queuen_preempt; // offset: 22  (0x16)
+    uint32_t sdmax_queuen_dummy_reg; // offset: 23  (0x17)
+    uint32_t sdmax_queuen_rb_wptr_poll_addr_lo; // offset: 24  (0x18)
+    uint32_t sdmax_queuen_rb_wptr_poll_addr_hi; // offset: 25  (0x19)
+    uint32_t sdmax_queuen_rb_aql_cntl; // offset: 26  (0x1A)
+    uint32_t sdmax_queuen_minor_ptr_update; // offset: 27  (0x1B)
+    uint32_t sdmax_queuen_mcu_dbg0; // offset: 28  (0x1C)
+    uint32_t sdmax_queuen_mcu_dbg1; // offset: 29  (0x1D)
+    uint32_t sdmax_queuen_context_switch_status; // offset: 30  (0x1E)
+    uint32_t sdmax_queuen_midcmd_cntl; // offset: 31  (0x1F)
+    uint32_t sdmax_queuen_midcmd_data0; // offset: 32  (0x20)
+    uint32_t sdmax_queuen_midcmd_data1; // offset: 33  (0x21)
+    uint32_t sdmax_queuen_midcmd_data2; // offset: 34  (0x22)
+    uint32_t sdmax_queuen_midcmd_data3; // offset: 35  (0x23)
+    uint32_t sdmax_queuen_midcmd_data4; // offset: 36  (0x24)
+    uint32_t sdmax_queuen_midcmd_data5; // offset: 37  (0x25)
+    uint32_t sdmax_queuen_midcmd_data6; // offset: 38  (0x26)
+    uint32_t sdmax_queuen_midcmd_data7; // offset: 39  (0x27)
+    uint32_t sdmax_queuen_midcmd_data8; // offset: 40  (0x28)
+    uint32_t sdmax_queuen_midcmd_data9; // offset: 41  (0x29)
+    uint32_t sdmax_queuen_midcmd_data10; // offset: 42  (0x2A)
+    uint32_t sdmax_queuen_utilization_lo; // offset: 43  (0x2B)
+    uint32_t sdmax_queuen_utilization_hi; // offset: 44  (0x2C)
+    uint32_t sdmax_queuen_wait_unsatisfied_thd; // offset: 45  (0x2D)
+    uint32_t sdmax_queuen_mqd_base_addr_lo; // offset: 46  (0x2E)
+    uint32_t sdmax_queuen_mqd_base_addr_hi; // offset: 47  (0x2F)
+    uint32_t sdmax_queuen_mqd_control; // offset: 48  (0x30)
+    uint32_t sdmax_queuen_context_status; // offset: 49  (0x31)
+    uint32_t sdmax_queuen_dummy_reg1; // offset: 50  (0x32)
+    uint32_t sdmax_queuen_dummy_reg2; // offset: 51  (0x33)
+    uint32_t reserved_52; // offset: 52  (0x34)
+    uint32_t reserved_53; // offset: 53  (0x35)
+    uint32_t reserved_54; // offset: 54  (0x36)
+    uint32_t reserved_55; // offset: 55  (0x37)
+    uint32_t reserved_56; // offset: 56  (0x38)
+    uint32_t reserved_57; // offset: 57  (0x39)
+    uint32_t reserved_58; // offset: 58  (0x3A)
+    uint32_t reserved_59; // offset: 59  (0x3B)
+    uint32_t reserved_60; // offset: 60  (0x3C)
+    uint32_t reserved_61; // offset: 61  (0x3D)
+    uint32_t reserved_62; // offset: 62  (0x3E)
+    uint32_t reserved_63; // offset: 63  (0x3F)
+    uint32_t reserved_64; // offset: 64  (0x40)
+    uint32_t reserved_65; // offset: 65  (0x41)
+    uint32_t reserved_66; // offset: 66  (0x42)
+    uint32_t reserved_67; // offset: 67  (0x43)
+    uint32_t reserved_68; // offset: 68  (0x44)
+    uint32_t reserved_69; // offset: 69  (0x45)
+    uint32_t reserved_70; // offset: 70  (0x46)
+    uint32_t reserved_71; // offset: 71  (0x47)
+    uint32_t reserved_72; // offset: 72  (0x48)
+    uint32_t reserved_73; // offset: 73  (0x49)
+    uint32_t reserved_74; // offset: 74  (0x4A)
+    uint32_t reserved_75; // offset: 75  (0x4B)
+    uint32_t reserved_76; // offset: 76  (0x4C)
+    uint32_t reserved_77; // offset: 77  (0x4D)
+    uint32_t reserved_78; // offset: 78  (0x4E)
+    uint32_t reserved_79; // offset: 79  (0x4F)
+    uint32_t reserved_80; // offset: 80  (0x50)
+    uint32_t reserved_81; // offset: 81  (0x51)
+    uint32_t reserved_82; // offset: 82  (0x52)
+    uint32_t reserved_83; // offset: 83  (0x53)
+    uint32_t reserved_84; // offset: 84  (0x54)
+    uint32_t reserved_85; // offset: 85  (0x55)
+    uint32_t reserved_86; // offset: 86  (0x56)
+    uint32_t reserved_87; // offset: 87  (0x57)
+    uint32_t reserved_88; // offset: 88  (0x58)
+    uint32_t reserved_89; // offset: 89  (0x59)
+    uint32_t reserved_90; // offset: 90  (0x5A)
+    uint32_t reserved_91; // offset: 91  (0x5B)
+    uint32_t reserved_92; // offset: 92  (0x5C)
+    uint32_t reserved_93; // offset: 93  (0x5D)
+    uint32_t reserved_94; // offset: 94  (0x5E)
+    uint32_t reserved_95; // offset: 95  (0x5F)
+    uint32_t reserved_96; // offset: 96  (0x60)
+    uint32_t reserved_97; // offset: 97  (0x61)
+    uint32_t reserved_98; // offset: 98  (0x62)
+    uint32_t reserved_99; // offset: 99  (0x63)
+    uint32_t reserved_100; // offset: 100  (0x64)
+    uint32_t reserved_101; // offset: 101  (0x65)
+    uint32_t reserved_102; // offset: 102  (0x66)
+    uint32_t reserved_103; // offset: 103  (0x67)
+    uint32_t reserved_104; // offset: 104  (0x68)
+    uint32_t reserved_105; // offset: 105  (0x69)
+    uint32_t reserved_106; // offset: 106  (0x6A)
+    uint32_t reserved_107; // offset: 107  (0x6B)
+    uint32_t reserved_108; // offset: 108  (0x6C)
+    uint32_t reserved_109; // offset: 109  (0x6D)
+    uint32_t reserved_110; // offset: 110  (0x6E)
+    uint32_t reserved_111; // offset: 111  (0x6F)
+    uint32_t reserved_112; // offset: 112  (0x70)
+    uint32_t reserved_113; // offset: 113  (0x71)
+    uint32_t reserved_114; // offset: 114  (0x72)
+    uint32_t reserved_115; // offset: 115  (0x73)
+    uint32_t reserved_116; // offset: 116  (0x74)
+    uint32_t reserved_117; // offset: 117  (0x75)
+    uint32_t reserved_118; // offset: 118  (0x76)
+    uint32_t reserved_119; // offset: 119  (0x77)
+    uint32_t reserved_120; // offset: 120  (0x78)
+    uint32_t reserved_121; // offset: 121  (0x79)
+    uint32_t reserved_122; // offset: 122  (0x7A)
+    uint32_t reserved_123; // offset: 123  (0x7B)
+    uint32_t reserved_124; // offset: 124  (0x7C)
+    uint32_t reserved_125; // offset: 125  (0x7D)
+	/* reserved_126,127: repurposed for driver-internal use */
+	uint32_t sdma_engine_id;
+	uint32_t sdma_queue_id;
+};
+
 struct v12_compute_mqd {
     uint32_t header; // offset: 0  (0x0)
     uint32_t compute_dispatch_initiator; // offset: 1  (0x1)
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
index df5bbdab1ea3..75d8bb4868e8 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
@@ -1082,6 +1082,91 @@ static void amdgpu_dpm_exit_umd_state(struct amdgpu_device *adev)
 					       AMD_PG_STATE_GATE);
 }
 
+/**
+ * amdgpu_dpm_reapply_umd_profile - Restore the active UMD profile
+ * @adev: amdgpu device
+ *
+ * Resume and reset retain the selected DPM level in software, but the
+ * underlying GFX and PMFW state is lost. Reapply a retained UMD profile
+ * after the common resume path has restored its default CG and PG state.
+ */
+int amdgpu_dpm_reapply_umd_profile(struct amdgpu_device *adev)
+{
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+	enum amd_dpm_forced_level level;
+	int ret, fallback_ret;
+
+	if (!pp_funcs || !pp_funcs->reapply_performance_level ||
+	    !pp_funcs->force_performance_level)
+		return 0;
+
+	/* Serialize the retained level with the GFX and PMFW reprogramming. */
+	mutex_lock(&adev->pm.mutex);
+	if (adev->pm.dpm.thermal_active) {
+		dev_warn(adev->dev,
+			 "Cannot restore UMD profile while thermal control is active\n");
+		ret = 0;
+		goto unlock;
+	}
+
+	if (pp_funcs->get_performance_level)
+		level = pp_funcs->get_performance_level(adev->powerplay.pp_handle);
+	else
+		level = adev->pm.dpm.forced_level;
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
+		break;
+	default:
+		ret = 0;
+		goto unlock;
+	}
+
+	/* The active profile already owns the GFX PG reference. */
+	ret = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_GFX,
+					      AMD_CG_STATE_UNGATE);
+	if (ret)
+		goto fallback_auto;
+
+	ret = amdgpu_asic_update_umd_stable_pstate(adev, true);
+	if (ret)
+		goto fallback_auto;
+
+	ret = pp_funcs->reapply_performance_level(adev->powerplay.pp_handle,
+						     level);
+	if (ret)
+		goto fallback_auto;
+
+unlock:
+	mutex_unlock(&adev->pm.mutex);
+
+	return ret;
+
+fallback_auto:
+	dev_warn(adev->dev, "Failed to restore UMD profile state: %d\n", ret);
+
+	/* Restore normal CG before releasing the profile's PG reference. */
+	amdgpu_asic_update_umd_stable_pstate(adev, false);
+	amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_GFX,
+					       AMD_CG_STATE_GATE);
+	mutex_unlock(&adev->pm.mutex);
+
+	mutex_lock(&adev->pm.stable_pstate_ctx_lock);
+	fallback_ret = amdgpu_dpm_force_performance_level(adev,
+						   AMD_DPM_FORCED_LEVEL_AUTO);
+	if (fallback_ret)
+		dev_warn(adev->dev, "Failed to fall back to AUTO performance level: %d\n",
+			 fallback_ret);
+	else
+		adev->pm.stable_pstate_ctx = NULL;
+	mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
+
+	return 0;
+}
+
 int amdgpu_dpm_force_performance_level(struct amdgpu_device *adev,
 				       enum amd_dpm_forced_level level)
 {
@@ -2124,6 +2209,24 @@ u64 amdgpu_dpm_get_npm_cap(struct amdgpu_device *adev)
 	return cap;
 }
 
+int amdgpu_dpm_set_npm_limit(struct amdgpu_device *adev, uint32_t limit)
+{
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+	int ret;
+
+	if (!pp_funcs || !pp_funcs->set_npm_limit)
+		return -EOPNOTSUPP;
+
+	if (amdgpu_sriov_vf(adev))
+		return -EPERM;
+
+	mutex_lock(&adev->pm.mutex);
+	ret = pp_funcs->set_npm_limit(adev->powerplay.pp_handle, limit);
+	mutex_unlock(&adev->pm.mutex);
+
+	return ret;
+}
+
 /**
  * amdgpu_dpm_get_xcp_metrics - Retrieve metrics for a specific compute
  * partition
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index c6e3748bb11b..48c52ac0fc77 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -2211,8 +2211,10 @@ out:
 /**
  * DOC: cur_node_power_limit
  *
- * The amdgpu driver provides a sysfs API for retrieving current node power limit.
- * The file cur_node_power_limit is used for this.
+ * The amdgpu driver provides a sysfs API for reading and writing the current node
+ * power limit in watts. The file cur_node_power_limit is used for this. Writing a
+ * new limit sends SetNpmLimit to PMFW. The file is writable only when the ASIC
+ * reports the write capability bit for this attribute.
  */
 static ssize_t amdgpu_show_cur_node_power_limit(struct device *dev,
 						struct device_attribute *attr, char *buf)
@@ -2231,6 +2233,27 @@ static ssize_t amdgpu_show_cur_node_power_limit(struct device *dev,
 	return sysfs_emit(buf, "%u\n", nplimit);
 }
 
+static ssize_t amdgpu_store_cur_node_power_limit(struct device *dev,
+						 struct device_attribute *attr,
+						 const char *buf, size_t count)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	u32 limit;
+	int r;
+
+	r = amdgpu_pm_get_access_if_active(adev);
+	if (r)
+		return r;
+
+	r = kstrtou32(buf, 0, &limit);
+	if (!r)
+		r = amdgpu_dpm_set_npm_limit(adev, limit);
+
+	amdgpu_pm_put_access(adev);
+	return r ? r : count;
+}
+
 /**
  * DOC: node_power
  *
@@ -2259,7 +2282,8 @@ static ssize_t amdgpu_show_node_power(struct device *dev,
  *
  * The amdgpu driver provides a sysfs API for retrieving current node power management status.
  * The file npm_status is used for this. It shows the status as enabled or disabled based on
- * current node power value. If node power is zero, status is disabled else enabled.
+ * current node power or npm status value. If node power or npm status is zero, status is
+ * disabled else enabled.
  */
 static ssize_t amdgpu_show_npm_status(struct device *dev,
 				      struct device_attribute *attr, char *buf)
@@ -2371,7 +2395,8 @@ static ssize_t amdgpu_show_baseboard_power_limit(struct device *dev,
 
 static DEVICE_ATTR(baseboard_temp, 0444, amdgpu_get_baseboard_temp_metrics, NULL);
 static DEVICE_ATTR(gpuboard_temp, 0444, amdgpu_get_gpuboard_temp_metrics, NULL);
-static DEVICE_ATTR(cur_node_power_limit, 0444, amdgpu_show_cur_node_power_limit, NULL);
+static DEVICE_ATTR(cur_node_power_limit, 0644, amdgpu_show_cur_node_power_limit,
+		   amdgpu_store_cur_node_power_limit);
 static DEVICE_ATTR(node_power, 0444, amdgpu_show_node_power, NULL);
 static DEVICE_ATTR(global_ppt_resid, 0444, amdgpu_show_global_ppt_resid, NULL);
 static DEVICE_ATTR(max_node_power_limit, 0444, amdgpu_show_max_node_power_limit, NULL);
diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
index 64d68a905f86..ba8624d28f28 100644
--- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
@@ -484,6 +484,7 @@ void amdgpu_dpm_set_power_state(struct amdgpu_device *adev,
 enum amd_dpm_forced_level amdgpu_dpm_get_performance_level(struct amdgpu_device *adev);
 int amdgpu_dpm_force_performance_level(struct amdgpu_device *adev,
 				       enum amd_dpm_forced_level level);
+int amdgpu_dpm_reapply_umd_profile(struct amdgpu_device *adev);
 int amdgpu_dpm_get_pp_num_states(struct amdgpu_device *adev,
 				 struct pp_states_info *states);
 int amdgpu_dpm_dispatch_task(struct amdgpu_device *adev,
@@ -617,5 +618,6 @@ bool amdgpu_dpm_reset_vcn_is_supported(struct amdgpu_device *adev);
 bool amdgpu_dpm_is_temp_metrics_supported(struct amdgpu_device *adev,
 					  enum smu_temp_metric_type type);
 u64 amdgpu_dpm_get_npm_cap(struct amdgpu_device *adev);
+int amdgpu_dpm_set_npm_limit(struct amdgpu_device *adev, uint32_t limit);
 
 #endif
diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
index 239c64b8edca..7b0ff5a0993e 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
@@ -358,24 +358,46 @@ static void pp_dpm_en_umd_pstate(struct pp_hwmgr  *hwmgr,
 	}
 }
 
-static int pp_dpm_force_performance_level(void *handle,
-					enum amd_dpm_forced_level level)
+static int pp_dpm_apply_performance_level(struct pp_hwmgr *hwmgr,
+					  enum amd_dpm_forced_level level,
+					  bool reapply)
 {
-	struct pp_hwmgr *hwmgr = handle;
+	int ret;
 
 	if (!hwmgr || !hwmgr->pm_en)
 		return -EINVAL;
 
-	if (level == hwmgr->dpm_level)
+	if (!reapply && level == hwmgr->dpm_level)
 		return 0;
 
 	pp_dpm_en_umd_pstate(hwmgr, &level);
 	hwmgr->request_dpm_level = level;
-	hwmgr_handle_task(hwmgr, AMD_PP_TASK_READJUST_POWER_STATE, NULL);
+	ret = hwmgr_handle_task(hwmgr, AMD_PP_TASK_READJUST_POWER_STATE, NULL);
+	if (ret)
+		return ret;
+
+	ret = phm_force_dpm_levels(hwmgr, level);
+	if (ret)
+		return ret;
+
+	hwmgr->dpm_level = level;
 
 	return 0;
 }
 
+static int pp_dpm_force_performance_level(void *handle,
+					enum amd_dpm_forced_level level)
+{
+	return pp_dpm_apply_performance_level(handle, level, false);
+}
+
+static int
+pp_dpm_reapply_performance_level(void *handle,
+					 enum amd_dpm_forced_level level)
+{
+	return pp_dpm_apply_performance_level(handle, level, true);
+}
+
 static enum amd_dpm_forced_level pp_dpm_get_performance_level(
 								void *handle)
 {
@@ -1536,6 +1558,7 @@ static const struct amd_pm_funcs pp_dpm_funcs = {
 	.load_firmware = pp_dpm_load_fw,
 	.wait_for_fw_loading_complete = pp_dpm_fw_loading_complete,
 	.force_performance_level = pp_dpm_force_performance_level,
+	.reapply_performance_level = pp_dpm_reapply_performance_level,
 	.get_performance_level = pp_dpm_get_performance_level,
 	.get_current_power_state = pp_dpm_get_current_power_state,
 	.dispatch_tasks = pp_dpm_dispatch_tasks,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index a65aa2129845..a901f8535598 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -884,6 +884,9 @@ static int smu_early_init(struct amdgpu_ip_block *ip_block)
 	r = smu_set_funcs(adev);
 	if (r)
 		return r;
+
+	smu_feature_init(smu, SMU_FEATURE_NUM_DEFAULT);
+
 	return smu_init_microcode(smu);
 }
 
@@ -1417,7 +1420,6 @@ static int smu_sw_init(struct amdgpu_ip_block *ip_block)
 	int i, ret;
 
 	smu->pool_size = adev->pm.smu_prv_buffer_size;
-	smu_feature_init(smu, SMU_FEATURE_NUM_DEFAULT);
 
 	INIT_WORK(&smu->throttling_logging_work, smu_throttling_logging_work_fn);
 	INIT_WORK(&smu->interrupt_work, smu_interrupt_work_fn);
@@ -2366,6 +2368,13 @@ static int smu_set_powergating_state(struct amdgpu_ip_block *ip_block,
 	return 0;
 }
 
+static void smu_enter_umd_pstate(struct smu_context *smu)
+{
+	smu_gpo_control(smu, false);
+	smu_gfx_ulv_control(smu, false);
+	smu_deep_sleep_control(smu, false);
+}
+
 static int smu_enable_umd_pstate(void *handle,
 		      enum amd_dpm_forced_level *level)
 {
@@ -2384,9 +2393,7 @@ static int smu_enable_umd_pstate(void *handle,
 		/* enter umd pstate, save current level, disable gfx cg*/
 		if (*level & profile_mode_mask) {
 			smu_dpm_ctx->saved_dpm_level = smu_dpm_ctx->dpm_level;
-			smu_gpo_control(smu, false);
-			smu_gfx_ulv_control(smu, false);
-			smu_deep_sleep_control(smu, false);
+			smu_enter_umd_pstate(smu);
 			amdgpu_asic_update_umd_stable_pstate(smu->adev, true);
 		}
 	} else {
@@ -2445,7 +2452,8 @@ static void smu_power_profile_mode_put(struct smu_context *smu,
 
 static int smu_adjust_power_state_dynamic(struct smu_context *smu,
 					  enum amd_dpm_forced_level level,
-					  bool skip_display_settings)
+					  bool skip_display_settings,
+					  bool force_performance_level)
 {
 	int ret = 0;
 	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
@@ -2472,7 +2480,7 @@ static int smu_adjust_power_state_dynamic(struct smu_context *smu,
 		}
 	}
 
-	if (smu_dpm_ctx->dpm_level != level) {
+	if (force_performance_level || smu_dpm_ctx->dpm_level != level) {
 		ret = smu_asic_set_performance_level(smu, level);
 		if (ret) {
 			if (ret == -EOPNOTSUPP)
@@ -2495,6 +2503,39 @@ static int smu_adjust_power_state_dynamic(struct smu_context *smu,
 	return ret;
 }
 
+static int
+smu_reapply_power_state_dynamic(struct smu_context *smu,
+				enum amd_dpm_forced_level level)
+{
+	struct smu_dpm_context *smu_dpm_ctx = &smu->smu_dpm;
+	int ret;
+
+	ret = smu_apply_clocks_adjust_rules(smu);
+	if (ret) {
+		dev_err(smu->adev->dev, "Failed to apply clocks adjust rules!");
+		return ret;
+	}
+
+	ret = smu_asic_set_performance_level(smu, level);
+	if (ret) {
+		if (ret == -EOPNOTSUPP)
+			dev_info(smu->adev->dev, "set performance level %d not supported",
+				 level);
+		else
+			dev_err(smu->adev->dev, "Failed to set performance level %d",
+				level);
+		return ret;
+	}
+
+	smu_dpm_ctx->dpm_level = level;
+
+	if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL ||
+	    smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM)
+		return 0;
+
+	return smu_bump_power_profile_mode(smu, NULL, 0);
+}
+
 static int smu_handle_task(struct smu_context *smu,
 			   enum amd_dpm_forced_level level,
 			   enum amd_pp_task task_id)
@@ -2509,13 +2550,13 @@ static int smu_handle_task(struct smu_context *smu,
 		ret = smu_pre_display_config_changed(smu);
 		if (ret)
 			return ret;
-		ret = smu_adjust_power_state_dynamic(smu, level, false);
+		ret = smu_adjust_power_state_dynamic(smu, level, false, false);
 		break;
 	case AMD_PP_TASK_COMPLETE_INIT:
-		ret = smu_adjust_power_state_dynamic(smu, level, true);
+		ret = smu_adjust_power_state_dynamic(smu, level, true, false);
 		break;
 	case AMD_PP_TASK_READJUST_POWER_STATE:
-		ret = smu_adjust_power_state_dynamic(smu, level, true);
+		ret = smu_adjust_power_state_dynamic(smu, level, true, false);
 		break;
 	default:
 		break;
@@ -2644,6 +2685,37 @@ static int smu_force_performance_level(void *handle,
 	return ret;
 }
 
+static int
+smu_reapply_performance_level(void *handle,
+			      enum amd_dpm_forced_level level)
+{
+	struct smu_context *smu = handle;
+	struct smu_dpm_context *smu_dpm_ctx = &smu->smu_dpm;
+
+	if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled)
+		return -EOPNOTSUPP;
+
+	if (!smu->is_apu && !smu_dpm_ctx->dpm_context)
+		return -EINVAL;
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (smu_dpm_ctx->dpm_level != level)
+		return -EINVAL;
+
+	smu_enter_umd_pstate(smu);
+
+	return smu_reapply_power_state_dynamic(smu, level);
+}
+
 static int smu_set_display_count(void *handle, uint32_t count)
 {
 	struct smu_context *smu = handle;
@@ -3995,6 +4067,19 @@ static u64 smu_get_npm_cap(void *handle)
 	return smu->ppt_funcs->get_npm_cap(smu);
 }
 
+static int smu_set_npm_limit(void *handle, uint32_t limit)
+{
+	struct smu_context *smu = handle;
+
+	if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled)
+		return -EOPNOTSUPP;
+
+	if (!smu->ppt_funcs || !smu->ppt_funcs->set_npm_limit)
+		return -EOPNOTSUPP;
+
+	return smu->ppt_funcs->set_npm_limit(smu, limit);
+}
+
 static ssize_t smu_sys_get_xcp_metrics(void *handle, int xcp_id, void *table)
 {
 	struct smu_context *smu = handle;
@@ -4017,6 +4102,7 @@ static const struct amd_pm_funcs swsmu_pm_funcs = {
 	.force_clock_level       = smu_force_ppclk_levels,
 	.emit_clock_levels       = smu_emit_ppclk_levels,
 	.force_performance_level = smu_force_performance_level,
+	.reapply_performance_level = smu_reapply_performance_level,
 	.read_sensor             = smu_read_sensor,
 	.get_apu_thermal_limit       = smu_get_apu_thermal_limit,
 	.set_apu_thermal_limit       = smu_set_apu_thermal_limit,
@@ -4069,6 +4155,7 @@ static const struct amd_pm_funcs swsmu_pm_funcs = {
 	.get_xcp_metrics                  = smu_sys_get_xcp_metrics,
 	.get_temp_metrics             = smu_sys_get_temp_metrics,
 	.temp_metrics_is_supported      = smu_temp_metrics_is_supported,
+	.set_npm_limit                = smu_set_npm_limit,
 	.get_npm_cap                  = smu_get_npm_cap,
 };
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index 1e5fc6ff76c8..9d47d2ab1b49 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
@@ -433,6 +433,8 @@ struct smu_table_context {
 	void                            *boot_overdrive_table;
 	void				*user_overdrive_table;
 
+	uint32_t			user_od_feature_mask;
+
 	struct smu_driver_table driver_tables[SMU_DRIVER_TABLE_COUNT];
 };
 
@@ -1626,6 +1628,12 @@ struct pptable_funcs {
 	 * unsupported.
 	 */
 	u64 (*get_npm_cap)(struct smu_context *smu);
+
+	/**
+	 * @set_npm_limit: Set node power limit in watts.
+	 */
+	int (*set_npm_limit)(struct smu_context *smu, uint32_t limit);
+
 	/**
 	 * @ras_send_msg: Send a message with a parameter from Ras
 	 * &msg: Type of message.
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_pmfw.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_pmfw.h
index a3401c4cc20b..61d76ebfdbd4 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_pmfw.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_pmfw.h
@@ -328,7 +328,7 @@ typedef struct __attribute__((packed, aligned(4))) {
   uint64_t GfxclkLowUtilizationAcc[8];
 } MetricsTable_t;
 
-#define SMU_SYSTEM_METRICS_TABLE_VERSION 0x1
+#define SMU_SYSTEM_METRICS_TABLE_VERSION 0x2
 
 #pragma pack(push, 4)
 typedef struct {
@@ -343,9 +343,13 @@ typedef struct {
   //NPM: NODE POWER MANAGEMENT
   uint32_t NodePowerLimit;
   uint32_t NodePower;
-  uint32_t GlobalPPTResidencyAcc;
+  uint32_t NPMResidencyAcc;
 
   uint16_t SystemPower[SYSTEM_POWER_MAX_ENTRIES];           // UBB Current Power and Power Threshold
+
+  // NPM flags
+  uint32_t NpmSupported;
+  uint32_t NpmEnabled;
 } SystemMetricsTable_t;
 #pragma pack(pop)
 
@@ -368,7 +372,7 @@ typedef struct __attribute__((aligned(4))) {
   uint8_t  FruId[PRODUCT_FRU_ID_LEN];
 } FRUProductInfo_t;
 
-#define SMU_STATIC_METRICS_TABLE_VERSION 0x1
+#define SMU_STATIC_METRICS_TABLE_VERSION 0x2
 
 #pragma pack(push, 4)
 typedef struct {
@@ -421,6 +425,8 @@ typedef struct {
   uint32_t PPT1Max;
   uint32_t PPT1Min;
   uint32_t PPT1Default;
+
+  uint32_t MaxNodePowerLimit;
 } StaticMetricsTable_t;
 #pragma pack(pop)
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_ppsmc.h
index 7ffb445f4c0c..888261c26409 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_ppsmc.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v15_0_8_ppsmc.h
@@ -83,7 +83,10 @@
 #define PPSMC_MSG_SetSoftMaxGl2clk	                0x32
 #define PPSMC_MSG_SetSoftMinFclk	                0x33
 #define PPSMC_MSG_SetSoftMaxFclk	                0x34
-#define PPSMC_Message_Count                         0x35
+/* NPM limit setter; NPM reads are served from the SystemMetricsTable. */
+#define PPSMC_MSG_SetNpmLimit                       0x40
+#define PPSMC_MSG_PrepareNodeForUnload              0x41
+#define PPSMC_Message_Count                         0x42
 
 /* PSMC Reset Types for driver msg argument */
 #define PPSMC_RESET_TYPE_DRIVER_MODE_1_RESET        0x1
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
index 2631e12b5e59..31b4ef025351 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
@@ -305,7 +305,9 @@
 	__SMU_DUMMY_MAP(SetSoftMaxFclk), \
 	__SMU_DUMMY_MAP(SetSoftMaxGl2clk), \
 	__SMU_DUMMY_MAP(SetSoftMinGl2clk), \
-	__SMU_DUMMY_MAP(GetSystemMetricsVersion),
+	__SMU_DUMMY_MAP(GetSystemMetricsVersion), \
+	__SMU_DUMMY_MAP(SetNpmLimit), \
+	__SMU_DUMMY_MAP(PrepareNodeForUnload),
 
 #undef __SMU_DUMMY_MAP
 #define __SMU_DUMMY_MAP(type)	SMU_MSG_##type
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
index 0c97bee2c4bb..f443000ba657 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
@@ -89,6 +89,7 @@ struct smu_15_0_dpm_context {
 	uint32_t                    workload_policy_mask;
 	uint32_t                    dcef_min_ds_clk;
 	uint64_t                    caps;
+	bool			    node_unload;
 	uint32_t		    board_volt;
 };
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
index f8ca5eb9adc6..2d05088619f6 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
@@ -1439,15 +1439,15 @@ static int navi10_force_clk_levels(struct smu_context *smu,
 
 		ret = smu_v11_0_get_dpm_freq_by_index(smu, clk_type, soft_min_level, &min_freq);
 		if (ret)
-			return 0;
+			return ret;
 
 		ret = smu_v11_0_get_dpm_freq_by_index(smu, clk_type, soft_max_level, &max_freq);
 		if (ret)
-			return 0;
+			return ret;
 
 		ret = smu_v11_0_set_soft_freq_limited_range(smu, clk_type, min_freq, max_freq, false);
 		if (ret)
-			return 0;
+			return ret;
 		break;
 	case SMU_DCEFCLK:
 		dev_info(smu->adev->dev, "Setting DCEFCLK min/max dpm level is not supported!\n");
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
index b42897e7c066..4ef3863fae1a 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
@@ -1460,7 +1460,7 @@ static int sienna_cichlid_force_clk_levels(struct smu_context *smu,
 	}
 
 forec_level_out:
-	return 0;
+	return ret;
 }
 
 static int sienna_cichlid_populate_umd_state_clk(struct smu_context *smu)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index 6beafabf1306..a9bf686783e9 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -1569,7 +1569,6 @@ static int smu_v13_0_0_od_edit_dpm_table(struct smu_context *smu,
 	struct amdgpu_device *adev = smu->adev;
 	uint32_t offset_of_voltageoffset;
 	int32_t minimum, maximum;
-	uint32_t feature_ctrlmask;
 	int i, ret = 0;
 
 	switch (type) {
@@ -1891,21 +1890,16 @@ static int smu_v13_0_0_od_edit_dpm_table(struct smu_context *smu,
 			if (ret)
 				return ret;
 		} else {
-			feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask;
 			memcpy(od_table,
 		       table_context->boot_overdrive_table,
 		       sizeof(OverDriveTableExternal_t));
-			od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask;
+			od_table->OverDriveTable.FeatureCtrlMask = 0;
+			table_context->user_od_feature_mask = 0;
 		}
 		fallthrough;
 	case PP_OD_COMMIT_DPM_TABLE:
-		/*
-		 * The member below instructs PMFW the settings focused in
-		 * this single operation.
-		 * `uint32_t FeatureCtrlMask;`
-		 * It does not contain actual informations about user's custom
-		 * settings. Thus we do not cache it.
-		 */
+		table_context->user_od_feature_mask =
+			od_table->OverDriveTable.FeatureCtrlMask;
 		offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary);
 		if (memcmp((u8 *)od_table + offset_of_voltageoffset,
 			   table_context->user_overdrive_table + offset_of_voltageoffset,
@@ -1918,7 +1912,6 @@ static int smu_v13_0_0_od_edit_dpm_table(struct smu_context *smu,
 				return ret;
 			}
 
-			od_table->OverDriveTable.FeatureCtrlMask = 0;
 			memcpy(table_context->user_overdrive_table + offset_of_voltageoffset,
 			       (u8 *)od_table + offset_of_voltageoffset,
 			       sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset);
@@ -2093,9 +2086,11 @@ static ssize_t smu_v13_0_0_get_gpu_metrics(struct smu_context *smu,
 	gpu_metrics->average_socket_power = metrics->AverageSocketPower;
 
 	if ((mp1_ver == IP_VERSION(13, 0, 0) &&
-	     (smu->smc_fw_version <= 0x004e1e00 || smu->smc_fw_version >= 0x004e8600)) ||
+	     (smu->smc_fw_version <= 0x004e1e00 ||
+	      smu->smc_fw_version >= 0x004e8600)) ||
 	    (mp1_ver == IP_VERSION(13, 0, 10) &&
-	     smu->smc_fw_version <= 0x00500800))
+	     (smu->smc_fw_version <= 0x00500800 ||
+	      smu->smc_fw_version >= 0x00505d00)))
 		gpu_metrics->energy_accumulator = metrics->EnergyAccumulator;
 
 	if (metrics->AverageGfxActivity <= SMU_13_0_0_BUSY_THRESHOLD)
@@ -2201,6 +2196,7 @@ static int smu_v13_0_0_set_default_od_settings(struct smu_context *smu)
 		       boot_od_table,
 		       sizeof(OverDriveTableExternal_t));
 		smu->user_dpm_profile.user_od = false;
+		smu->smu_table.user_od_feature_mask = 0;
 	} else if (smu->user_dpm_profile.user_od) {
 		memcpy(&user_od_table_bak,
 		       user_od_table,
@@ -2250,12 +2246,8 @@ static int smu_v13_0_0_restore_user_od_settings(struct smu_context *smu)
 	OverDriveTableExternal_t *od_table = table_context->overdrive_table;
 	OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table;
 	int res;
-
-	user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) |
-							BIT(PP_OD_FEATURE_UCLK_BIT) |
-							BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) |
-							BIT(PP_OD_FEATURE_FAN_CURVE_BIT) |
-							BIT(PP_OD_FEATURE_ZERO_FAN_BIT);
+	user_od_table->OverDriveTable.FeatureCtrlMask =
+		table_context->user_od_feature_mask;
 	res = smu_v13_0_0_upload_overdrive_table(smu, user_od_table);
 	user_od_table->OverDriveTable.FeatureCtrlMask = 0;
 	if (res == 0)
@@ -3069,8 +3061,10 @@ static int smu_v13_0_0_set_ppt_limit(struct smu_context *smu,
 
 		if (current_od_table.OverDriveTable.Ppt) {
 			od_table->OverDriveTable.Ppt = 0;
-			od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT;
-
+			table_context->user_od_feature_mask &=
+				~(1U << PP_OD_FEATURE_PPT_BIT);
+			od_table->OverDriveTable.FeatureCtrlMask =
+				table_context->user_od_feature_mask;
 			ret = smu_v13_0_0_upload_overdrive_table(smu, od_table);
 			if (ret) {
 				dev_err(smu->adev->dev, "Failed to upload overdrive table!\n");
@@ -3084,8 +3078,9 @@ static int smu_v13_0_0_set_ppt_limit(struct smu_context *smu,
 			return ret;
 
 		od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100;
-		od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT;
-
+		table_context->user_od_feature_mask |= 1U << PP_OD_FEATURE_PPT_BIT;
+		od_table->OverDriveTable.FeatureCtrlMask =
+			table_context->user_od_feature_mask;
 		ret = smu_v13_0_0_upload_overdrive_table(smu, od_table);
 		if (ret) {
 		  dev_err(smu->adev->dev, "Failed to upload overdrive table!\n");
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
index 5fe409a23772..d8715a94ec6a 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
@@ -1578,7 +1578,6 @@ static int smu_v13_0_7_od_edit_dpm_table(struct smu_context *smu,
 	struct amdgpu_device *adev = smu->adev;
 	uint32_t offset_of_voltageoffset;
 	int32_t minimum, maximum;
-	uint32_t feature_ctrlmask;
 	int i, ret = 0;
 
 	switch (type) {
@@ -1900,22 +1899,17 @@ static int smu_v13_0_7_od_edit_dpm_table(struct smu_context *smu,
 			if (ret)
 				return ret;
 		} else {
-			feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask;
 			memcpy(od_table,
 					table_context->boot_overdrive_table,
 					sizeof(OverDriveTableExternal_t));
-			od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask;
+			od_table->OverDriveTable.FeatureCtrlMask = 0;
+			table_context->user_od_feature_mask = 0;
 		}
 		fallthrough;
 
 	case PP_OD_COMMIT_DPM_TABLE:
-		/*
-		 * The member below instructs PMFW the settings focused in
-		 * this single operation.
-		 * `uint32_t FeatureCtrlMask;`
-		 * It does not contain actual informations about user's custom
-		 * settings. Thus we do not cache it.
-		 */
+		table_context->user_od_feature_mask =
+			od_table->OverDriveTable.FeatureCtrlMask;
 		offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary);
 		if (memcmp((u8 *)od_table + offset_of_voltageoffset,
 			   table_context->user_overdrive_table + offset_of_voltageoffset,
@@ -1928,7 +1922,6 @@ static int smu_v13_0_7_od_edit_dpm_table(struct smu_context *smu,
 				return ret;
 			}
 
-			od_table->OverDriveTable.FeatureCtrlMask = 0;
 			memcpy(table_context->user_overdrive_table + offset_of_voltageoffset,
 			       (u8 *)od_table + offset_of_voltageoffset,
 			       sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset);
@@ -2097,7 +2090,8 @@ static ssize_t smu_v13_0_7_get_gpu_metrics(struct smu_context *smu,
 					       metrics->Vcn1ActivityPercentage);
 
 	gpu_metrics->average_socket_power = metrics->AverageSocketPower;
-	if (smu->smc_fw_version <= 0x00521400)
+	if (smu->smc_fw_version <= 0x00521400 ||
+	    smu->smc_fw_version >= 0x00526200)
 		gpu_metrics->energy_accumulator = metrics->EnergyAccumulator;
 
 	if (metrics->AverageGfxActivity <= SMU_13_0_7_BUSY_THRESHOLD)
@@ -2203,6 +2197,7 @@ static int smu_v13_0_7_set_default_od_settings(struct smu_context *smu)
 		       boot_od_table,
 		       sizeof(OverDriveTableExternal_t));
 		smu->user_dpm_profile.user_od = false;
+		smu->smu_table.user_od_feature_mask = 0;
 	} else if (smu->user_dpm_profile.user_od) {
 		memcpy(&user_od_table_bak,
 		       user_od_table,
@@ -2253,11 +2248,8 @@ static int smu_v13_0_7_restore_user_od_settings(struct smu_context *smu)
 	OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table;
 	int res;
 
-	user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) |
-							BIT(PP_OD_FEATURE_UCLK_BIT) |
-							BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) |
-							BIT(PP_OD_FEATURE_FAN_CURVE_BIT) |
-							BIT(PP_OD_FEATURE_ZERO_FAN_BIT);
+	user_od_table->OverDriveTable.FeatureCtrlMask =
+		table_context->user_od_feature_mask;
 	res = smu_v13_0_7_upload_overdrive_table(smu, user_od_table);
 	user_od_table->OverDriveTable.FeatureCtrlMask = 0;
 	if (res == 0)
@@ -2444,7 +2436,21 @@ static int smu_v13_0_7_init_ppt_limits(struct smu_context *smu)
 static int smu_v13_0_7_get_power_profile_mode(struct smu_context *smu, char *buf)
 {
 	DpmActivityMonitorCoeffIntExternal_t *activity_monitor_external;
-	uint32_t i, j, size = 0;
+	DpmActivityMonitorCoeffInt_t *activity_monitor;
+	static const char * const title[] = {
+		"PROFILE_INDEX(NAME)",
+		"CLOCK_TYPE(NAME)",
+		"ActiveHystLimit",
+		"IdleHystLimit",
+		"FPS",
+		"MinActiveFreqType",
+		"MinActiveFreq",
+		"BoosterFreqType",
+		"BoosterFreq",
+		"PD_Data_limit_c",
+		"PD_Data_error_coeff",
+		"PD_Data_error_rate_coeff"};
+	u32 i, size = 0;
 	int16_t workload_type = 0;
 	int result = 0;
 
@@ -2456,12 +2462,9 @@ static int smu_v13_0_7_get_power_profile_mode(struct smu_context *smu, char *buf
 	if (!activity_monitor_external)
 		return -ENOMEM;
 
-	size += sysfs_emit_at(buf, size, "                              ");
-	for (i = 0; i <= PP_SMC_POWER_PROFILE_WINDOW3D; i++)
-		size += sysfs_emit_at(buf, size, "%d %-14s%s", i, amdgpu_pp_profile_name[i],
-			(i == smu->power_profile_mode) ? "* " : "  ");
-
-	size += sysfs_emit_at(buf, size, "\n");
+	size += sysfs_emit_at(buf, size, "%16s %s %s %s %s %s %s %s %s %s %s %s\n",
+		title[0], title[1], title[2], title[3], title[4], title[5],
+		title[6], title[7], title[8], title[9], title[10], title[11]);
 
 	for (i = 0; i <= PP_SMC_POWER_PROFILE_WINDOW3D; i++) {
 		/* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */
@@ -2482,31 +2485,50 @@ static int smu_v13_0_7_get_power_profile_mode(struct smu_context *smu, char *buf
 			dev_err(smu->adev->dev, "[%s] Failed to get activity monitor!", __func__);
 			goto out;
 		}
-	}
 
-#define PRINT_DPM_MONITOR(field)									\
-do {													\
-	size += sysfs_emit_at(buf, size, "%-30s", #field);						\
-	for (j = 0; j <= PP_SMC_POWER_PROFILE_WINDOW3D; j++)						\
-		size += sysfs_emit_at(buf, size, "%-18d", activity_monitor_external[j].DpmActivityMonitorCoeffInt.field);		\
-	size += sysfs_emit_at(buf, size, "\n");								\
-} while (0)
-
-	PRINT_DPM_MONITOR(Gfx_ActiveHystLimit);
-	PRINT_DPM_MONITOR(Gfx_IdleHystLimit);
-	PRINT_DPM_MONITOR(Gfx_FPS);
-	PRINT_DPM_MONITOR(Gfx_MinActiveFreqType);
-	PRINT_DPM_MONITOR(Gfx_BoosterFreqType);
-	PRINT_DPM_MONITOR(Gfx_MinActiveFreq);
-	PRINT_DPM_MONITOR(Gfx_BoosterFreq);
-	PRINT_DPM_MONITOR(Fclk_ActiveHystLimit);
-	PRINT_DPM_MONITOR(Fclk_IdleHystLimit);
-	PRINT_DPM_MONITOR(Fclk_FPS);
-	PRINT_DPM_MONITOR(Fclk_MinActiveFreqType);
-	PRINT_DPM_MONITOR(Fclk_BoosterFreqType);
-	PRINT_DPM_MONITOR(Fclk_MinActiveFreq);
-	PRINT_DPM_MONITOR(Fclk_BoosterFreq);
-#undef PRINT_DPM_MONITOR
+		activity_monitor =
+			&activity_monitor_external[i].DpmActivityMonitorCoeffInt;
+
+		size += sysfs_emit_at(buf, size, "%2d %14s%s:\n",
+			i, amdgpu_pp_profile_name[i],
+			(i == smu->power_profile_mode) ? "*" : " ");
+
+		size += sysfs_emit_at(buf, size,
+			"%19s %d(%13s) %7d %7d %7d %7d %7d",
+			" ",
+			0,
+			"GFXCLK",
+			activity_monitor->Gfx_ActiveHystLimit,
+			activity_monitor->Gfx_IdleHystLimit,
+			activity_monitor->Gfx_FPS,
+			activity_monitor->Gfx_MinActiveFreqType,
+			activity_monitor->Gfx_MinActiveFreq);
+
+		size += sysfs_emit_at(buf, size, "%7d %7d %7d %7d %7d\n",
+			activity_monitor->Gfx_BoosterFreqType,
+			activity_monitor->Gfx_BoosterFreq,
+			activity_monitor->Gfx_PD_Data_limit_c,
+			activity_monitor->Gfx_PD_Data_error_coeff,
+			activity_monitor->Gfx_PD_Data_error_rate_coeff);
+
+		size += sysfs_emit_at(buf, size,
+			"%19s %d(%13s) %7d %7d %7d %7d %7d",
+			" ",
+			1,
+			"FCLK",
+			activity_monitor->Fclk_ActiveHystLimit,
+			activity_monitor->Fclk_IdleHystLimit,
+			activity_monitor->Fclk_FPS,
+			activity_monitor->Fclk_MinActiveFreqType,
+			activity_monitor->Fclk_MinActiveFreq);
+
+		size += sysfs_emit_at(buf, size, "%7d %7d %7d %7d %7d\n",
+			activity_monitor->Fclk_BoosterFreqType,
+			activity_monitor->Fclk_BoosterFreq,
+			activity_monitor->Fclk_PD_Data_limit_c,
+			activity_monitor->Fclk_PD_Data_error_coeff,
+			activity_monitor->Fclk_PD_Data_error_rate_coeff);
+	}
 
 	result = size;
 out:
@@ -2689,7 +2711,10 @@ static int smu_v13_0_7_set_ppt_limit(struct smu_context *smu,
 
 		if (current_od_table.OverDriveTable.Ppt) {
 			od_table->OverDriveTable.Ppt = 0;
-			od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT;
+			table_context->user_od_feature_mask &=
+				~(1U << PP_OD_FEATURE_PPT_BIT);
+			od_table->OverDriveTable.FeatureCtrlMask =
+				table_context->user_od_feature_mask;
 
 			ret = smu_v13_0_7_upload_overdrive_table(smu, od_table);
 			if (ret) {
@@ -2704,7 +2729,9 @@ static int smu_v13_0_7_set_ppt_limit(struct smu_context *smu,
 			return ret;
 
 		od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100;
-		od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT;
+		table_context->user_od_feature_mask |= 1U << PP_OD_FEATURE_PPT_BIT;
+		od_table->OverDriveTable.FeatureCtrlMask =
+			table_context->user_od_feature_mask;
 
 		ret = smu_v13_0_7_upload_overdrive_table(smu, od_table);
 		if (ret) {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
index 6621e50de965..3f2c120ef2dc 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
@@ -2195,8 +2195,11 @@ static ssize_t smu_v14_0_2_get_gpu_metrics(struct smu_context *smu,
 					       metrics->Vcn1ActivityPercentage);
 
 	gpu_metrics->average_socket_power = metrics->AverageSocketPower;
-	if (mp1_ver == IP_VERSION(14, 0, 3) && smu->smc_fw_version >= 0x00685000)
-	    gpu_metrics->energy_accumulator = metrics->EnergyAccumulator;
+	if ((mp1_ver == IP_VERSION(14, 0, 2) &&
+	     smu->smc_fw_version >= 0x00664800) ||
+	    (mp1_ver == IP_VERSION(14, 0, 3) &&
+	     smu->smc_fw_version >= 0x00685000))
+		gpu_metrics->energy_accumulator = metrics->EnergyAccumulator;
 
 	if (metrics->AverageGfxActivity <= SMU_14_0_2_BUSY_THRESHOLD)
 		gpu_metrics->average_gfxclk_frequency = metrics->AverageGfxclkFrequencyPostDs;
@@ -2385,6 +2388,7 @@ static int smu_v14_0_2_set_default_od_settings(struct smu_context *smu)
 		       boot_od_table,
 		       sizeof(OverDriveTableExternal_t));
 		smu->user_dpm_profile.user_od = false;
+		smu->smu_table.user_od_feature_mask = 0;
 	} else if (smu->user_dpm_profile.user_od) {
 		memcpy(&user_od_table_bak,
 		       user_od_table,
@@ -2433,11 +2437,8 @@ static int smu_v14_0_2_restore_user_od_settings(struct smu_context *smu)
 	OverDriveTableExternal_t *user_od_table = table_context->user_overdrive_table;
 	int res;
 
-	user_od_table->OverDriveTable.FeatureCtrlMask = BIT(PP_OD_FEATURE_GFXCLK_BIT) |
-							BIT(PP_OD_FEATURE_UCLK_BIT) |
-							BIT(PP_OD_FEATURE_GFX_VF_CURVE_BIT) |
-							BIT(PP_OD_FEATURE_FAN_CURVE_BIT) |
-							BIT(PP_OD_FEATURE_ZERO_FAN_BIT);
+	user_od_table->OverDriveTable.FeatureCtrlMask =
+		table_context->user_od_feature_mask;
 	res = smu_v14_0_2_upload_overdrive_table(smu, user_od_table);
 	user_od_table->OverDriveTable.FeatureCtrlMask = 0;
 	if (res == 0)
@@ -2522,7 +2523,6 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu,
 	struct amdgpu_device *adev = smu->adev;
 	uint32_t offset_of_voltageoffset;
 	int32_t minimum, maximum;
-	uint32_t feature_ctrlmask;
 	int i, ret = 0;
 
 	switch (type) {
@@ -2811,21 +2811,26 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu,
 			if (ret)
 				return ret;
 		} else {
-			feature_ctrlmask = od_table->OverDriveTable.FeatureCtrlMask;
 			memcpy(od_table,
 		       table_context->boot_overdrive_table,
 		       sizeof(OverDriveTableExternal_t));
-			od_table->OverDriveTable.FeatureCtrlMask = feature_ctrlmask;
+			od_table->OverDriveTable.FeatureCtrlMask = 0;
+			table_context->user_od_feature_mask = 0;
 		}
 		fallthrough;
 	case PP_OD_COMMIT_DPM_TABLE:
 		/*
-		 * The member below instructs PMFW the settings focused in
-		 * this single operation.
-		 * `uint32_t FeatureCtrlMask;`
-		 * It does not contain actual informations about user's custom
-		 * settings. Thus we do not cache it.
+		 * PMFW interprets FeatureCtrlMask as the complete set of OD
+		 * features that are currently overridden, not as a delta
+		 * describing this single operation. Any feature whose bit is
+		 * clear gets reset to its default on import. od_table's mask
+		 * has accumulated every feature the user touched since the
+		 * last full restore, so record it and keep restating it on
+		 * subsequent uploads.
 		 */
+		table_context->user_od_feature_mask =
+			od_table->OverDriveTable.FeatureCtrlMask;
+
 		offset_of_voltageoffset = offsetof(OverDriveTable_t, VoltageOffsetPerZoneBoundary);
 		if (memcmp((u8 *)od_table + offset_of_voltageoffset,
 			   table_context->user_overdrive_table + offset_of_voltageoffset,
@@ -2838,7 +2843,6 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu,
 				return ret;
 			}
 
-			od_table->OverDriveTable.FeatureCtrlMask = 0;
 			memcpy(table_context->user_overdrive_table + offset_of_voltageoffset,
 			       (u8 *)od_table + offset_of_voltageoffset,
 			       sizeof(OverDriveTableExternal_t) - offset_of_voltageoffset);
@@ -2882,7 +2886,10 @@ static int smu_v14_0_2_set_ppt_limit(struct smu_context *smu,
 
 		if (current_od_table.OverDriveTable.Ppt) {
 			od_table->OverDriveTable.Ppt = 0;
-			od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT;
+			table_context->user_od_feature_mask &=
+				~(1U << PP_OD_FEATURE_PPT_BIT);
+			od_table->OverDriveTable.FeatureCtrlMask =
+				table_context->user_od_feature_mask;
 
 			ret = smu_v14_0_2_upload_overdrive_table(smu, od_table);
 			if (ret) {
@@ -2897,7 +2904,9 @@ static int smu_v14_0_2_set_ppt_limit(struct smu_context *smu,
 			return ret;
 
 		od_table->OverDriveTable.Ppt = (limit * 100) / msg_limit - 100;
-		od_table->OverDriveTable.FeatureCtrlMask |= 1U << PP_OD_FEATURE_PPT_BIT;
+		table_context->user_od_feature_mask |= 1U << PP_OD_FEATURE_PPT_BIT;
+		od_table->OverDriveTable.FeatureCtrlMask =
+			table_context->user_od_feature_mask;
 
 		ret = smu_v14_0_2_upload_overdrive_table(smu, od_table);
 		if (ret) {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
index 9e42c27f60ee..ce56d540e85e 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
@@ -25,6 +25,7 @@
 
 #include <linux/firmware.h>
 #include "amdgpu.h"
+#include "amdgpu_xgmi.h"
 #include "amdgpu_smu.h"
 #include "smu_v15_0_8_pmfw.h"
 #include "smu15_driver_if_v15_0_8.h"
@@ -137,6 +138,8 @@ static const struct cmn2asic_msg_mapping smu_v15_0_8_message_map[SMU_MSG_MAX_COU
 	MSG_MAP(SetSoftMaxGl2clk,                    PPSMC_MSG_SetSoftMaxGl2clk,		0),
 	MSG_MAP(SetSoftMinFclk,                      PPSMC_MSG_SetSoftMinFclk,			0),
 	MSG_MAP(SetSoftMaxFclk,                      PPSMC_MSG_SetSoftMaxFclk,			0),
+	MSG_MAP(SetNpmLimit,                         PPSMC_MSG_SetNpmLimit,			0),
+	MSG_MAP(PrepareNodeForUnload,                PPSMC_MSG_PrepareNodeForUnload,		0),
 };
 
 /* TODO: Update the clk map once enum PPCLK is updated in smu15_driver_if_v15_0_8.h */
@@ -588,19 +591,99 @@ static int smu_v15_0_8_get_system_metrics_table(struct smu_context *smu)
 	return 0;
 }
 
+static inline void smu_v15_0_8_cap_set(struct smu_context *smu, u32 cap)
+{
+	struct smu_15_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+
+	dpm_context->caps |= BIT_ULL(cap);
+}
+
+static bool smu_v15_0_8_cap_supported(struct smu_context *smu, u32 cap)
+{
+	struct smu_15_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+
+	return !!(dpm_context->caps & BIT_ULL(cap));
+}
+
+static void smu_v15_0_8_init_npm_cap(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *tables = smu_table->tables;
+	SystemMetricsTable_t *metrics;
+	struct smu_table *sys_table;
+	uint32_t table_version;
+	int ret;
+
+	if (smu->adev->gmc.xgmi.physical_node_id != 0)
+		return;
+
+	ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetSystemMetricsVersion,
+				   &table_version);
+	if (ret)
+		return;
+
+	if (table_version < SMU_SYSTEM_METRICS_TABLE_VERSION)
+		return;
+
+	ret = smu_v15_0_8_get_system_metrics_table(smu);
+	if (ret)
+		return;
+
+	sys_table = &tables[SMU_TABLE_PMFW_SYSTEM_METRICS];
+	metrics = (SystemMetricsTable_t *)sys_table->cache.buffer;
+	if (!metrics->NpmSupported)
+		return;
+
+	smu_v15_0_8_cap_set(smu, SMU_15_0_8_CAP_NPM);
+}
+
+static void smu_v15_0_8_init_node_unload_cap(struct smu_context *smu)
+{
+	uint32_t fw_ver = smu->smc_fw_version;
+	uint32_t pgm = (fw_ver >> 24) & 0xFF;
+
+	if (smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NODE_UNLOAD))
+		return;
+
+	if (pgm == 2 && fw_ver >= SMU_15_0_8_NODE_UNLOAD_MIN_FW_VERSION)
+		smu_v15_0_8_cap_set(smu, SMU_15_0_8_CAP_NODE_UNLOAD);
+}
+
+static void smu_v15_0_8_init_caps(struct smu_context *smu)
+{
+	smu_v15_0_8_init_npm_cap(smu);
+	smu_v15_0_8_init_node_unload_cap(smu);
+}
+
+static u64 smu_v15_0_8_get_npm_cap(struct smu_context *smu)
+{
+	if (!smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NPM))
+		return 0;
+
+	return AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_CUR_NODE_POWER_LIMIT) |
+	       AMDGPU_NPM_CAP_W(AMDGPU_NPM_CAP_CUR_NODE_POWER_LIMIT) |
+	       AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_NODE_POWER)           |
+	       AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_GLOBAL_PPT_RESID)     |
+	       AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_MAX_NODE_POWER_LIMIT) |
+	       AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_NPM_STATUS);
+}
+
 static int smu_v15_0_8_get_npm_data(struct smu_context *smu,
 				    enum amd_pp_sensors sensor,
 				    uint32_t *value)
 {
 	struct smu_table_context *smu_table = &smu->smu_table;
+	PPTable_t *pptable = (PPTable_t *)smu_table->driver_pptable;
 	struct smu_table *tables = smu_table->tables;
 	SystemMetricsTable_t *metrics;
 	struct smu_table *sys_table;
 	int ret;
 
+	if (!smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NPM))
+		return -EOPNOTSUPP;
+
 	if (sensor == AMDGPU_PP_SENSOR_MAXNODEPOWERLIMIT) {
-		/*TBD as of now put 0 */
-		*value = 0;
+		*value = pptable->MaxNodePowerLimit;
 		return 0;
 	}
 
@@ -619,10 +702,10 @@ static int smu_v15_0_8_get_npm_data(struct smu_context *smu,
 		*value = SMUQ10_ROUND(metrics->NodePower);
 		break;
 	case AMDGPU_PP_SENSOR_NPMSTATUS:
-		*value = !!SMUQ10_ROUND(metrics->NodePower);
+		*value = !!metrics->NpmEnabled;
 		break;
 	case AMDGPU_PP_SENSOR_GPPTRESIDENCY:
-		*value = SMUQ10_ROUND(metrics->GlobalPPTResidencyAcc);
+		*value = SMUQ10_ROUND(metrics->NPMResidencyAcc);
 		break;
 	default:
 		return -EINVAL;
@@ -631,6 +714,15 @@ static int smu_v15_0_8_get_npm_data(struct smu_context *smu,
 	return 0;
 }
 
+static int smu_v15_0_8_set_npm_limit(struct smu_context *smu, uint32_t limit)
+{
+	if (!smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NPM))
+		return -EOPNOTSUPP;
+
+	return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetNpmLimit,
+					       limit, NULL);
+}
+
 static int smu_v15_0_8_read_sensor(struct smu_context *smu,
 				   enum amd_pp_sensors sensor, void *data,
 				   uint32_t *size)
@@ -1149,6 +1241,10 @@ static int smu_v15_0_8_set_driver_pptable(struct smu_context *smu)
 				static_metrics->pldmVersion[0];
 		dpm_context->board_volt = static_metrics->InputTelemetryVoltageInmV;
 		smu_v15_0_8_init_xgmi_data(smu, static_metrics);
+		smu_v15_0_8_init_caps(smu);
+		if (smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NPM))
+			pptable->MaxNodePowerLimit =
+				static_metrics->MaxNodePowerLimit;
 		pptable->init = true;
 	}
 
@@ -1285,8 +1381,46 @@ static int smu_v15_0_8_register_irq_handler(struct smu_context *smu)
 	return ret;
 }
 
+static void smu_v15_0_8_notify_node_unload(struct smu_context *smu)
+{
+	if (!smu_v15_0_8_cap_supported(smu, SMU_15_0_8_CAP_NODE_UNLOAD))
+		return;
+
+	smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PrepareNodeForUnload, 0, NULL);
+}
+
+static void smu_v15_0_8_mark_node_unload(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	struct amdgpu_hive_info *hive;
+	struct amdgpu_device *peer;
+	struct smu_context *peer_smu;
+	struct smu_15_0_dpm_context *dpm;
+
+	hive = amdgpu_get_xgmi_hive(adev);
+	if (!hive) {
+		dpm = smu->smu_dpm.dpm_context;
+		if (dpm)
+			dpm->node_unload = true;
+		return;
+	}
+
+	list_for_each_entry(peer, &hive->device_list, gmc.xgmi.head) {
+		peer_smu = peer->powerplay.pp_handle;
+		if (!peer_smu)
+			continue;
+		dpm = peer_smu->smu_dpm.dpm_context;
+		if (dpm)
+			dpm->node_unload = true;
+	}
+
+	amdgpu_put_xgmi_hive(hive);
+}
+
 static int smu_v15_0_8_notify_unload(struct smu_context *smu)
 {
+	struct smu_context *primary_smu;
+	struct smu_15_0_dpm_context *dpm = smu->smu_dpm.dpm_context;
 	struct smu_msg_ctl *ctl = &smu->msg_ctl;
 	struct smu_msg_args args = {
 		.msg     = SMU_MSG_PrepareMp1ForUnload,
@@ -1296,6 +1430,13 @@ static int smu_v15_0_8_notify_unload(struct smu_context *smu)
 	if (amdgpu_in_reset(smu->adev))
 		return 0;
 
+	if (dpm && !dpm->node_unload) {
+		primary_smu = smu_get_primary_dev(smu);
+		if (primary_smu)
+			smu_v15_0_8_notify_node_unload(primary_smu);
+		smu_v15_0_8_mark_node_unload(smu);
+	}
+
 	dev_dbg(smu->adev->dev, "Notify PMFW about driver unload");
 	/* Ignore return, just intimate FW that driver is not going to be there */
 	ctl->ops->send_msg(ctl, &args);
@@ -2382,6 +2523,8 @@ static const struct pptable_funcs smu_v15_0_8_ppt_funcs = {
 	.get_unique_id = smu_v15_0_8_get_unique_id,
 	.get_ppt_limit = smu_v15_0_8_get_ppt_limit,
 	.set_ppt_limit = smu_v15_0_8_set_ppt_limit,
+	.get_npm_cap = smu_v15_0_8_get_npm_cap,
+	.set_npm_limit = smu_v15_0_8_set_npm_limit,
 	.emit_clk_levels = smu_v15_0_8_emit_clk_levels,
 	.read_sensor = smu_v15_0_8_read_sensor,
 	.populate_umd_state_clk = smu_v15_0_8_populate_umd_state_clk,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
index 46fce0a79c21..dff601bc0f9a 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
@@ -32,6 +32,9 @@
 #define SMU_15_0_8_MAX_AID 2
 #define SMU_15_0_8_MAX_MID 2
 #define SMU_15_0_8_MAX_HBM_STACKS 12
+#define SMU_15_0_8_CAP_NPM			1
+#define SMU_15_0_8_CAP_NODE_UNLOAD		2
+#define SMU_15_0_8_NODE_UNLOAD_MIN_FW_VERSION	0x027d0c00u /* 2.125.12.0 */
 extern void smu_v15_0_8_set_ppt_funcs(struct smu_context *smu);
 
 typedef struct {
@@ -61,6 +64,7 @@ typedef struct {
 	uint32_t PPT1Max;
 	uint32_t PPT1Min;
 	uint32_t PPT1Default;
+	uint32_t MaxNodePowerLimit;
 	bool init;
 } PPTable_t;
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index 2bd3ea17e789..b0abc2f504c3 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -27,6 +27,32 @@
 #include "smu_cmn.h"
 #include "soc15_common.h"
 
+#include "amdgpu_xgmi.h"
+
+struct smu_context *smu_get_primary_dev(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	struct amdgpu_hive_info *hive;
+	struct amdgpu_device *peer;
+
+	if (adev->gmc.xgmi.physical_node_id == 0)
+		return smu;
+
+	hive = amdgpu_get_xgmi_hive(adev);
+	if (!hive)
+		return NULL;
+
+	list_for_each_entry(peer, &hive->device_list, gmc.xgmi.head) {
+		if (peer->gmc.xgmi.physical_node_id == 0) {
+			amdgpu_put_xgmi_hive(hive);
+			return peer->powerplay.pp_handle;
+		}
+	}
+
+	amdgpu_put_xgmi_hive(hive);
+	return NULL;
+}
+
 /*
  * DO NOT use these for err/warn/info/debug messages.
  * Use dev_err, dev_warn, dev_info and dev_dbg instead.
@@ -1387,11 +1413,12 @@ int smu_cmn_print_dpm_clk_levels(struct smu_context *smu,
 				 uint32_t cur_clk, char *buf, int *offset)
 {
 	uint32_t min_clk, max_clk, level_index, count;
+	uint32_t nearest_level = 0, nearest_diff = U32_MAX;
 	uint32_t freq_values[3];
+	uint32_t clk_diff, level_clk;
 	int size, lvl, i;
 	bool is_fine_grained;
 	bool is_deep_sleep;
-	bool freq_match;
 
 	if (!dpm_table || !buf)
 		return -EINVAL;
@@ -1403,23 +1430,41 @@ int smu_cmn_print_dpm_clk_levels(struct smu_context *smu,
 	min_clk = SMU_DPM_TABLE_MIN(dpm_table);
 	max_clk = SMU_DPM_TABLE_MAX(dpm_table);
 
-	/* Deep sleep - current clock < min_clock/2, TBD: cur_clk = 0 as GFXOFF */
-	is_deep_sleep = cur_clk < min_clk / 2;
+	/* A clock below DPM0 is reported as a special deep-sleep level. */
+	is_deep_sleep = cur_clk < min_clk;
 	if (is_deep_sleep) {
 		size += sysfs_emit_at(buf, size, "S: %uMhz *\n", cur_clk);
 		level_index = 1;
 	}
 
+	/*
+	 * Discrete tables mark the level nearest to the current clock.  Clamp
+	 * clocks above the advertised range to the last level.  Fine-grained
+	 * tables keep reporting the actual current clock between min and max.
+	 */
 	if (!is_fine_grained || count == 1) {
+		if (!is_deep_sleep) {
+			if (cur_clk >= max_clk) {
+				nearest_level = count - 1;
+			} else {
+				for (i = 0; i < count; i++) {
+					level_clk = dpm_table->dpm_levels[i].value;
+					clk_diff = cur_clk > level_clk ?
+						   cur_clk - level_clk : level_clk - cur_clk;
+					if (clk_diff < nearest_diff) {
+						nearest_diff = clk_diff;
+						nearest_level = i;
+					}
+				}
+			}
+		}
+
 		for (i = 0; i < count; i++) {
-			freq_match = !is_deep_sleep &&
-				     smu_cmn_freqs_match(
-					     cur_clk,
-					     dpm_table->dpm_levels[i].value);
 			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
 					      level_index + i,
 					      dpm_table->dpm_levels[i].value,
-					      freq_match ? "*" : "");
+					      (!is_deep_sleep &&
+					       i == nearest_level) ? "*" : "");
 		}
 	} else {
 		count = 2;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
index ae6742f5298f..6d7f47c934f2 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
@@ -249,6 +249,8 @@ int smu_cmn_dpm_pcie_gen_idx(int gen);
 int smu_cmn_dpm_pcie_width_idx(int width);
 int smu_cmn_check_fw_version(struct smu_context *smu);
 
+struct smu_context *smu_get_primary_dev(struct smu_context *smu);
+
 int smu_cmn_get_pptable_from_firmware(struct smu_context *smu, void **table,
 				      uint32_t *size, uint32_t pptable_id);
 
diff --git a/drivers/gpu/drm/amd/ras/core/aca_v1_0.c b/drivers/gpu/drm/amd/ras/core/aca_v1_0.c
index a4145f30f45f..f4fe95a8eca3 100644
--- a/drivers/gpu/drm/amd/ras/core/aca_v1_0.c
+++ b/drivers/gpu/drm/amd/ras/core/aca_v1_0.c
@@ -353,6 +353,12 @@ static const struct aca_block_info aca_v1_0_mmhub = {
 	},
 };
 
+/* used to set feature mask without parse error info */
+static const struct aca_block_info aca_v1_0_pcie_bif = {
+	.name = "pcie_bif",
+	.ras_block_id = RAS_BLOCK_ID__PCIE_BIF,
+};
+
 static const struct aca_block_info aca_v1_0_xgmi = {
 	.name = "xgmi",
 	.ras_block_id = RAS_BLOCK_ID__XGMI_WAFL,
@@ -369,6 +375,7 @@ static const struct aca_block_info *aca_block_info_v1_0[] = {
 	&aca_v1_0_gfx,
 	&aca_v1_0_sdma,
 	&aca_v1_0_mmhub,
+	&aca_v1_0_pcie_bif,
 	&aca_v1_0_xgmi,
 };
 
diff --git a/drivers/gpu/drm/amd/ras/core/eeprom.c b/drivers/gpu/drm/amd/ras/core/eeprom.c
index efa04cd8fbe0..40430f0b4d2e 100644
--- a/drivers/gpu/drm/amd/ras/core/eeprom.c
+++ b/drivers/gpu/drm/amd/ras/core/eeprom.c
@@ -693,7 +693,7 @@ static int ras_eeprom_update_header(struct ras_eeprom_control *control)
 	record_count = ras_umc_get_saved_eeprom_count(ras_core);
 	/* Modify the header if it exceeds.
 	 */
-	if (record_count >= control->record_threshold_count) {
+	if (record_count > control->record_threshold_count) {
 		control->tbl_hdr.header = RAS_TABLE_HDR_BAD;
 		if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1) {
 			control->tbl_rai.rma_status = RAS_GPU_RETIRED__ECC_REACH_THRESHOLD;
diff --git a/drivers/gpu/drm/amd/ras/core/ras.h b/drivers/gpu/drm/amd/ras/core/ras.h
index 73e3960c7c8e..c2977d2c9ba7 100644
--- a/drivers/gpu/drm/amd/ras/core/ras.h
+++ b/drivers/gpu/drm/amd/ras/core/ras.h
@@ -60,7 +60,7 @@ enum ras_gpu_health_status {
 };
 
 enum ras_core_fw_feature_flags {
-	RAS_CORE_FW_FEATURE_BIT__RAS_EEPROM = BIT_ULL(0),
+	RAS_CORE_FW_FEATURE_BIT__RAS_EEPROM = 0,
 };
 
 enum ras_block_id {
@@ -408,6 +408,7 @@ struct ras_core_context {
 	bool poison_supported;
 
 	bool is_rma;
+	bool bad_page_over_threshold;
 	bool is_initialized;
 	bool in_early_init;
 	bool early_init_service_enabled;
diff --git a/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c b/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c
index fe0e35357af1..223e0a791fae 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c
@@ -26,6 +26,7 @@
 #include "ras.h"
 #include "eeprom.h"
 #include "eeprom_fw.h"
+#include "ras_mp1.h"
 
 static bool __ras_eeprom_disabled(struct ras_core_context *ras_core)
 {
@@ -36,25 +37,36 @@ static const struct ras_eeprom_ops *ras_eeprom_mgr_get_ip_func(
 				struct ras_core_context *ras_core, uint32_t ip_version)
 {
 	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	int ret = 0;
+	u32 status = 0;
+
+	ret = ras_mp1_check_feature_status(ras_core,
+		(enum ras_mp1_feature_id)RAS_CORE_FW_FEATURE_BIT__RAS_EEPROM, &status);
+	if (ret)
+		RAS_DEV_ERR(ras_core->dev, "Check RAS EEPROM feature failed, ret=%d", ret);
 
 	switch (ip_version) {
 	case IP_VERSION(13, 0, 6):
-	case IP_VERSION(13, 0, 14):
 	case IP_VERSION(13, 0, 12):
+	case IP_VERSION(13, 0, 14):
 	case IP_VERSION(13, 0, 15):
 		mgr->eeprom_early_init_service_supported = false;
-		return &ras_drv_eeprom_ops;
+		break;
 	case IP_VERSION(15, 0, 8):
 		mgr->eeprom_early_init_service_supported = true;
-		mgr->fw_record_enabled = true;
-		return &ras_fw_eeprom_ops;
+		break;
 	default:
 		RAS_DEV_ERR(ras_core->dev,
 			"EEPROM(MP1) ip version(0x%x) is not supported!\n", ip_version);
-		break;
+		return NULL;
 	}
 
-	return NULL;
+	if (status) {
+		mgr->fw_record_enabled = true;
+		return &ras_fw_eeprom_ops;
+	} else {
+		return &ras_drv_eeprom_ops;
+	}
 }
 
 int ras_eeprom_mgr_sw_init(struct ras_core_context *ras_core)
@@ -261,6 +273,7 @@ int ras_eeprom_mgr_check_and_report_status(struct ras_core_context *ras_core, bo
 {
 	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
 	struct ras_eeprom_info info = {0};
+	u32 bad_page_count, bad_page_threshold;
 	int valid_err_count;
 	int ret = 0;
 
@@ -275,25 +288,32 @@ int ras_eeprom_mgr_check_and_report_status(struct ras_core_context *ras_core, bo
 		return ret;
 
 	valid_err_count = ras_umc_get_saved_eeprom_count(ras_core);
+	bad_page_count = ras_umc_get_badpage_count(ras_core);
+	bad_page_threshold = mgr->ras_err_threshold * ras_core->ras_umc.max_pages_per_row;
 	if (info.eeprom_status == RAS_EEPROM_OK) {
 		RAS_DEV_DBG(ras_core->dev,
-			"Found existing EEPROM table with %d records\n",
-			valid_err_count);
+			"Found existing EEPROM table with %u records\n",
+			bad_page_count);
 
 		/* Warn if error count reaches or exceeds 90% of the threshold */
 		if (10 * valid_err_count >= 9 * mgr->ras_err_threshold)
 			RAS_DEV_WARN(ras_core->dev,
-				"RAS records:%u exceeds 90%% of threshold:%d\n",
-				valid_err_count, mgr->ras_err_threshold);
+				"RAS records:%u exceeds 90%% of threshold:%u\n",
+				bad_page_count, bad_page_threshold);
 
 	} else if (info.eeprom_status == RAS_EEPROM_LOCKED) {
-		if (info.rma_status || (valid_err_count >= mgr->ras_err_threshold)) {
+		if (info.rma_status || valid_err_count > mgr->ras_err_threshold) {
 			if (info.rma_status)
-				RAS_DEV_ERR(ras_core->dev, "RAS records:%d exceed error threshold",
-					valid_err_count);
+				RAS_DEV_ERR(ras_core->dev,
+					    "RAS records:%u exceed error threshold",
+					    bad_page_count);
 			else
-				RAS_DEV_ERR(ras_core->dev, "RAS records:%d exceed threshold:%d",
-					valid_err_count, mgr->ras_err_threshold);
+				RAS_DEV_ERR(ras_core->dev,
+					    "RAS records:%u exceed threshold:%u",
+					    bad_page_count, bad_page_threshold);
+
+			if (mgr->work_mode_over_thresh != RAS_WORK_MODE_OVER_THRESH_DEBUG)
+				ras_core->bad_page_over_threshold = true;
 
 			if (mgr->work_mode_over_thresh == RAS_WORK_MODE_OVER_THRESH_STRICT) {
 				RAS_DEV_WARN(ras_core->dev,
@@ -303,27 +323,28 @@ int ras_eeprom_mgr_check_and_report_status(struct ras_core_context *ras_core, bo
 				RAS_DEV_ERR(ras_core->dev,
 				"User defined threshold is set, runtime service will be halt when threshold is reached\n");
 			}
-		} else if ((valid_err_count < mgr->ras_err_threshold) &&
+		} else if ((valid_err_count <= mgr->ras_err_threshold) &&
 				mgr->eeprom_ops->unlock_eeprom) {
 			/* This means that, the threshold was increased since
 			 * the last time the system was booted, and now,
-			 * the saved error count is less than threshold,
+			 * the saved error count does not exceed the threshold,
 			 * so that at least one more record can be saved,
-			 * before the page count threshold is reached.
+			 * before the page count threshold is exceeded.
 			 */
 			ret = mgr->eeprom_ops->unlock_eeprom(ras_core);
 			if (ret) {
 				RAS_DEV_ERR(ras_core->dev, "Failed to resize RAS table\n");
 			} else {
 				ras_core->is_rma = false;
+				ras_core->bad_page_over_threshold = false;
 				RAS_DEV_INFO(ras_core->dev,
-					"records:%d threshold:%d, resetting RAS table header signature",
-					valid_err_count, mgr->ras_err_threshold);
+					"records:%u threshold:%u, resetting RAS table header signature",
+					bad_page_count, bad_page_threshold);
 			}
 		} else {
 			RAS_DEV_ERR(ras_core->dev,
-				"records:%d threshold:%d, not support to resize RAS table\n",
-				valid_err_count, mgr->ras_err_threshold);
+				"records:%u threshold:%u, not support to resize RAS table\n",
+				bad_page_count, bad_page_threshold);
 		}
 	} else {
 		RAS_DEV_ERR(ras_core->dev, "Error ras eeprom status:0x%x\n", info.eeprom_status);
diff --git a/drivers/gpu/drm/amd/ras/core/ras_psp.c b/drivers/gpu/drm/amd/ras/core/ras_psp.c
index c16acf4b57f7..5854c662cdfb 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_psp.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_psp.c
@@ -430,7 +430,8 @@ static int __check_ras_ta_cmd_resp(struct ras_core_context *ras_core,
 }
 
 static int send_ras_ta_runtime_cmd(struct ras_core_context *ras_core,
-			enum ras_ta_cmd_id cmd_id, void *in, uint32_t in_size,
+			uint32_t session_id, enum ras_ta_cmd_id cmd_id,
+			void *in, uint32_t in_size,
 			void *out, uint32_t out_size)
 {
 	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
@@ -471,7 +472,7 @@ static int send_ras_ta_runtime_cmd(struct ras_core_context *ras_core,
 	memcpy(&ras_cmd->ras_in_message, in, in_size);
 
 	invoke_cmd.ta_cmd_id = cmd_id;
-	invoke_cmd.session_id = ta_ctx->session_id;
+	invoke_cmd.session_id = session_id;
 
 	ret = send_psp_cmd(ras_core, GFX_CMD_ID_INVOKE_CMD,
 			&invoke_cmd, sizeof(invoke_cmd), &resp);
@@ -498,10 +499,15 @@ static int send_ras_ta_runtime_cmd(struct ras_core_context *ras_core,
 		goto unlock;
 	}
 
-	if (!ras_cmd->ras_status && out && out_size)
+	/* Hand the TA output back regardless of status; ret carries the status. */
+	if (out && out_size)
 		memcpy(out, &ras_cmd->ras_out_message, out_size);
 
-	ret = __check_ras_ta_cmd_resp(ras_core, ras_cmd);
+	/* cmd from user space requires special handling for CHECK_ACA_BANK_ERROR_TYPES */
+	if (cmd_id == RAS_TA_CMD_ID__CHECK_ACA_BANK_ERROR_TYPES)
+		__check_ras_ta_cmd_resp(ras_core, ras_cmd);
+	else
+		ret = __check_ras_ta_cmd_resp(ras_core, ras_cmd);
 
 unlock:
 	mutex_unlock(&ta_ctx->ta_mutex);
@@ -540,7 +546,9 @@ static int trigger_ras_ta_error(struct ras_core_context *ras_core,
 	dev_mask &= RAS_TA_INST_MASK;
 	info->sub_block_index |= dev_mask;
 
-	return send_ras_ta_runtime_cmd(ras_core, RAS_TA_CMD_ID__TRIGGER_ERROR,
+	return send_ras_ta_runtime_cmd(ras_core,
+				ras_core->ras_psp.ta_ctx.session_id,
+				RAS_TA_CMD_ID__TRIGGER_ERROR,
 				info, sizeof(*info), NULL, 0);
 }
 
@@ -789,6 +797,21 @@ int ras_psp_unsideload_ras_ta(struct ras_core_context *ras_core,
 	return unload_ras_ta_fw(ras_core, ras_ta_unload);
 }
 
+int ras_psp_invoke_ras_ta(struct ras_core_context *ras_core,
+		uint32_t session_id, uint32_t cmd_id,
+		void *shared_buf, uint32_t shared_buf_len)
+{
+	/* The debugfs caller hands over a full ras_ta_cmd shared buffer. */
+	struct ras_ta_cmd *ras_cmd = shared_buf;
+
+	if (!shared_buf || shared_buf_len < sizeof(*ras_cmd))
+		return -EINVAL;
+
+	return send_ras_ta_runtime_cmd(ras_core, session_id, cmd_id,
+			&ras_cmd->ras_in_message, sizeof(ras_cmd->ras_in_message),
+			&ras_cmd->ras_out_message, sizeof(ras_cmd->ras_out_message));
+}
+
 int ras_psp_reload_firmwares(struct ras_core_context *ras_core,
 		uint32_t flags)
 {
@@ -814,7 +837,7 @@ int ras_psp_enable_features(struct ras_core_context *ras_core,
 		return -ENOEXEC;
 	}
 
-	return send_ras_ta_runtime_cmd(ras_core,
+	return send_ras_ta_runtime_cmd(ras_core, ta_ctx->session_id,
 			enable ? RAS_TA_CMD_ID__ENABLE_FEATURES :
 				 RAS_TA_CMD_ID__DISABLE_FEATURES,
 			info, sizeof(*info), NULL, 0);
@@ -850,8 +873,9 @@ int ras_psp_query_address(struct ras_core_context *ras_core,
 	if (!addr_in || !addr_out)
 		return -EINVAL;
 
-	return send_ras_ta_runtime_cmd(ras_core, RAS_TA_CMD_ID__QUERY_ADDRESS,
-		addr_in, sizeof(*addr_in), addr_out, sizeof(*addr_out));
+	return send_ras_ta_runtime_cmd(ras_core, ta_ctx->session_id,
+		RAS_TA_CMD_ID__QUERY_ADDRESS, addr_in, sizeof(*addr_in),
+		addr_out, sizeof(*addr_out));
 }
 
 int ras_psp_sw_init(struct ras_core_context *ras_core)
diff --git a/drivers/gpu/drm/amd/ras/core/ras_psp.h b/drivers/gpu/drm/amd/ras/core/ras_psp.h
index 4cc133c6a6d3..91d17fdd6bda 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_psp.h
+++ b/drivers/gpu/drm/amd/ras/core/ras_psp.h
@@ -196,6 +196,9 @@ int ras_psp_sideload_ras_ta(struct ras_core_context *ras_core,
 		struct ras_psp_ta_load *ta_load);
 int ras_psp_unsideload_ras_ta(struct ras_core_context *ras_core,
 		struct ras_psp_ta_unload *ras_ta_unload);
+int ras_psp_invoke_ras_ta(struct ras_core_context *ras_core,
+		uint32_t session_id, uint32_t cmd_id,
+		void *shared_buf, uint32_t shared_buf_len);
 int ras_psp_enable_features(struct ras_core_context *ras_core,
 	struct ras_ta_enable_features_input *info, bool enable);
 int ras_psp_trigger_error(struct ras_core_context *ras_core,
diff --git a/drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c b/drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c
index 49ba57f58878..bdd3da2c66f3 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c
@@ -80,13 +80,23 @@ static int ras_psp_v13_0_get_ras_block_maps(struct ras_core_context *ras_core,
 static int ras_psp_v13_0_get_ras_hw_caps(struct ras_core_context *ras_core,
 				struct ras_hw_caps *ras_cap)
 {
-	uint32_t ras_hw_cap, bit;
+	uint32_t ras_hw_cap = 0, bit;
 	uint32_t i;
 
 	if (!ras_cap)
 		return -EINVAL;
 
-	ras_hw_cap = RAS_DEV_RREG32_SOC15(ras_core->dev, MP0, 0, regMP0_SMN_C2PMSG_127);
+	/*
+	 * driver only manages a few IP block RAS fearure
+	 * when GPU is connected CPU through XGMI
+	 */
+	if (RAS_DEV_IS_APP_APU(ras_core->dev)) {
+		ras_hw_cap |= (1 << AMDGPU_RAS_BLOCK__GFX |
+					   1 << AMDGPU_RAS_BLOCK__SDMA |
+					   1 << AMDGPU_RAS_BLOCK__MMHUB);
+	} else {
+		ras_hw_cap = RAS_DEV_RREG32_SOC15(ras_core->dev, MP0, 0, regMP0_SMN_C2PMSG_127);
+	}
 
 	ras_cap->poison_supported = true;
 	ras_cap->flex_mca_enabled = false;
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc.c b/drivers/gpu/drm/amd/ras/core/ras_umc.c
index aaf98cbee619..db4a0d525091 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc.c
@@ -705,7 +705,7 @@ void ras_umc_report_badpage_info(struct ras_core_context *ras_core)
 			&ras_umc->last_channel_bitmap);
 	}
 
-	if (ras_core->is_rma)
+	if (ras_core->is_rma || ras_core->bad_page_over_threshold)
 		ras_core_event_notify(ras_core, RAS_EVENT_ID__DEVICE_RMA, NULL);
 }
 
@@ -852,6 +852,13 @@ static int ras_umc_save_bad_pages(struct ras_core_context *ras_core)
 		RAS_DEV_INFO(ras_core->dev, "Saved %d pages to EEPROM table.\n",
 			ras_umc_count_valid_pages(ras_core,
 				&data->bps[eeprom_record_num], save_count));
+
+		/* The table is only tagged as over threshold while the new
+		 * records are written out, so the status has to be
+		 * re-evaluated here.
+		 */
+		ras_eeprom_mgr_check_and_report_status(ras_core, true);
+		ras_umc_report_badpage_info(ras_core);
 	}
 
 exit:
diff --git a/drivers/gpu/drm/amd/ras/core/ta_if.h b/drivers/gpu/drm/amd/ras/core/ta_if.h
index 30c058a524db..87d5aa3f8477 100644
--- a/drivers/gpu/drm/amd/ras/core/ta_if.h
+++ b/drivers/gpu/drm/amd/ras/core/ta_if.h
@@ -44,6 +44,7 @@ enum ras_ta_cmd_id {
 	RAS_TA_CMD_ID__QUERY_BLOCK_INFO,
 	RAS_TA_CMD_ID__QUERY_SUB_BLOCK_INFO,
 	RAS_TA_CMD_ID__QUERY_ADDRESS,
+	RAS_TA_CMD_ID__CHECK_ACA_BANK_ERROR_TYPES = 10,
 	MAX_RAS_TA_CMD_ID
 };
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
index 58a4d16bdeee..30f2312e3700 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
@@ -543,6 +543,9 @@ static int amdgpu_ras_mgr_resume(struct amdgpu_ip_block *ip_block)
 	if (!ras_mgr->ras_is_ready)
 		return 0;
 
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
 	ret = ras_psp_reload_firmwares(ras_mgr->ras_core, 0);
 	if (ret)
 		RAS_DEV_ERR(adev,
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h b/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
index 1c5e02de7fca..00ff12ccb01a 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
@@ -106,6 +106,12 @@
 	amdgpu_ip_map_logical_to_dev_mask(&adev->ip_map, ip##_HWIP, mask); \
 })
 
+#define RAS_DEV_IS_APP_APU(dev) \
+({ \
+	struct amdgpu_device *adev = (struct amdgpu_device *)dev; \
+	adev->gmc.is_app_apu; \
+})
+
 static inline void *ras_radix_tree_delete_iter(struct radix_tree_root *root, void *iter)
 {
 	return radix_tree_delete(root, ((struct radix_tree_iter *)iter)->index);