+ amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_aca.o amdgpu_dev_coredump.o \

+ amdgpu_cper.o

+#include "amdgpu_cper.h"

+void amdgpu_bios_release(struct amdgpu_device *adev);

+ /* CPER */

+ struct amdgpu_cper cper;

+

+ bool ras_default_ecc_enabled;

+ bool debug_disable_gpu_ring_reset;

+ /* Protection for the following isolation structure */

+ bool enforce_isolation[MAX_XCP];

+ struct amdgpu_isolation {

+ void *owner;

+ struct dma_fence *spearhead;

+ struct amdgpu_sync active;

+ struct amdgpu_sync prev;

+ } isolation[MAX_XCP];

+

+ /* This flag is used to determine how VRAM allocations are handled for APUs

+ * in KFD: VRAM or GTT.

+ */

+ bool apu_prefer_gtt;

+struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,

+ struct amdgpu_ring *ring,

+ struct amdgpu_job *job);

+ {"CONFIG", ACA_REG_IDX_CONFIG},

+ bank.smu_err_type = type;

+static void aca_banks_generate_cper(struct amdgpu_device *adev,

+ enum aca_smu_type type,

+ struct aca_banks *banks,

+ int count)

+{

+ struct aca_bank_node *node;

+ struct aca_bank *bank;

+ int r;

+

+ if (!adev->cper.enabled)

+ return;

+

+ if (!banks || !count) {

+ dev_warn(adev->dev, "fail to generate cper records\n");

+ return;

+ }

+

+ /* UEs must be encoded into separate CPER entries */

+ if (type == ACA_SMU_TYPE_UE) {

+ struct aca_banks de_banks;

+

+ aca_banks_init(&de_banks);

+ list_for_each_entry(node, &banks->list, node) {

+ bank = &node->bank;

+ if (bank->aca_err_type == ACA_ERROR_TYPE_DEFERRED) {

+ r = aca_banks_add_bank(&de_banks, bank);

+ if (r)

+ dev_warn(adev->dev, "fail to add de banks, ret = %d\n", r);

+ } else {

+ if (amdgpu_cper_generate_ue_record(adev, bank))

+ dev_warn(adev->dev, "fail to generate ue cper records\n");

+ }

+ }

+

+ if (!list_empty(&de_banks.list)) {

+ if (amdgpu_cper_generate_ce_records(adev, &de_banks, de_banks.nr_banks))

+ dev_warn(adev->dev, "fail to generate de cper records\n");

+ }

+

+ aca_banks_release(&de_banks);

+ } else {

+ /*

+ * SMU_TYPE_CE banks are combined into 1 CPER entries,

+ * they could be CEs or DEs or both

+ */

+ if (amdgpu_cper_generate_ce_records(adev, banks, count))

+ dev_warn(adev->dev, "fail to generate ce cper records\n");

+ }

+}

+

+ aca_banks_generate_cper(adev, type, &banks, count);

+

+ /* DEs may contain in CEs or UEs */

+ if (type != ACA_ERROR_TYPE_DEFERRED)

+ aca_log_aca_error(handle, ACA_ERROR_TYPE_DEFERRED, err_data);

+

+#define ACA_BANK_ERR_IS_DEFFERED(bank) \

+ (ACA_REG__STATUS__POISON((bank)->regs[ACA_REG_IDX_STATUS]) || \

+ ACA_REG__STATUS__DEFERRED((bank)->regs[ACA_REG_IDX_STATUS]))

+

+#define ACA_BANK_ERR_CE_DE_DECODE(bank) \

+ (ACA_BANK_ERR_IS_DEFFERED(bank) ? ACA_ERROR_TYPE_DEFERRED : \

+ ACA_ERROR_TYPE_CE)

+

+#define ACA_BANK_ERR_UE_DE_DECODE(bank) \

+ (ACA_BANK_ERR_IS_DEFFERED(bank) ? ACA_ERROR_TYPE_DEFERRED : \

+ ACA_ERROR_TYPE_UE)

+

+ ACA_REG_IDX_CONFIG = 4,

+ ACA_SMU_TYPE_INVALID = -1,

+struct aca_hwip {

+ int hwid;

+ int mcatype;

+};

+

+ enum aca_error_type aca_err_type;

+ enum aca_smu_type smu_err_type;

+struct aca_banks {

+ int nr_banks;

+ struct list_head list;

+};

+

+static bool acp_is_idle(struct amdgpu_ip_block *ip_block)

+ atif->backlight_caps.data_points = characteristics.number_of_points;

+ memcpy(atif->backlight_caps.luminance_data,

+ characteristics.data_points,

+ sizeof(atif->backlight_caps.luminance_data));

+ memcpy(caps, &atif->backlight_caps, sizeof(*caps));

+ } else if (adev->apu_prefer_gtt) {

+ } else if (adev->apu_prefer_gtt) {

+struct kfd_process_device;

+void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo);

+void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo)

+ .build_dequeue_wait_counts_packet_info = kgd_gfx_v9_build_dequeue_wait_counts_packet_info,

+ .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell

+ .build_dequeue_wait_counts_packet_info = kgd_gfx_v9_build_dequeue_wait_counts_packet_info,

+ .hqd_reset = kgd_gfx_v9_hqd_reset,

+ .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell

+static uint32_t kgd_gfx_v9_4_3_hqd_sdma_get_doorbell(struct amdgpu_device *adev,

+ int engine, int queue)

+{

+ uint32_t reg_offset = get_sdma_rlc_reg_offset(adev, engine, queue);

+ uint32_t status = RREG32(regSDMA_RLC0_CONTEXT_STATUS + reg_offset);

+ uint32_t doorbell_off = RREG32(regSDMA_RLC0_DOORBELL_OFFSET + reg_offset);

+ bool is_active = !!REG_GET_FIELD(status, SDMA_RLC0_CONTEXT_STATUS, SELECTED);

+

+ return is_active ? doorbell_off >> 2 : 0;

+}

+

+ .build_dequeue_wait_counts_packet_info =

+ kgd_gfx_v9_build_dequeue_wait_counts_packet_info,

+ .hqd_reset = kgd_gfx_v9_hqd_reset,

+ .hqd_sdma_get_doorbell = kgd_gfx_v9_4_3_hqd_sdma_get_doorbell

+void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,

+ uint32_t sch_wave,

+ uint32_t que_sleep,

+ if (sch_wave)

+ *reg_data = REG_SET_FIELD(*reg_data,

+ CP_IQ_WAIT_TIME2,

+ SCH_WAVE,

+ sch_wave);

+ if (que_sleep)

+ *reg_data = REG_SET_FIELD(*reg_data,

+ CP_IQ_WAIT_TIME2,

+ QUE_SLEEP,

+ que_sleep);

+uint32_t kgd_gfx_v10_hqd_sdma_get_doorbell(struct amdgpu_device *adev,

+ int engine, int queue)

+{

+ return 0;

+}

+

+ .build_dequeue_wait_counts_packet_info = kgd_gfx_v10_build_dequeue_wait_counts_packet_info,

+ .hqd_reset = kgd_gfx_v10_hqd_reset,

+ .hqd_sdma_get_doorbell = kgd_gfx_v10_hqd_sdma_get_doorbell

+void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,

+ uint32_t sch_wave,

+ uint32_t que_sleep,

+uint32_t kgd_gfx_v10_hqd_sdma_get_doorbell(struct amdgpu_device *adev,

+ int engine, int queue);

+ .build_dequeue_wait_counts_packet_info = kgd_gfx_v10_build_dequeue_wait_counts_packet_info,

+ .hqd_reset = kgd_gfx_v10_hqd_reset,

+ .hqd_sdma_get_doorbell = kgd_gfx_v10_hqd_sdma_get_doorbell

+static uint32_t kgd_gfx_v11_hqd_sdma_get_doorbell(struct amdgpu_device *adev,

+ int engine, int queue)

+{

+ return 0;

+}

+

+ .hqd_reset = kgd_gfx_v11_hqd_reset,

+ .hqd_sdma_get_doorbell = kgd_gfx_v11_hqd_sdma_get_doorbell

+static uint32_t kgd_gfx_v12_hqd_sdma_get_doorbell(struct amdgpu_device *adev,

+ int engine, int queue)

+{

+ return 0;

+}

+

+ .hqd_sdma_get_doorbell = kgd_gfx_v12_hqd_sdma_get_doorbell

+void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,

+ uint32_t sch_wave,

+ uint32_t que_sleep,

+ if (sch_wave)

+ *reg_data = REG_SET_FIELD(*reg_data,

+ CP_IQ_WAIT_TIME2,

+ SCH_WAVE,

+ sch_wave);

+ if (que_sleep)

+ *reg_data = REG_SET_FIELD(*reg_data,

+ CP_IQ_WAIT_TIME2,

+ QUE_SLEEP,

+ que_sleep);

+uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev,

+ int engine, int queue)

+

+{

+ return 0;

+}

+

+ .build_dequeue_wait_counts_packet_info = kgd_gfx_v9_build_dequeue_wait_counts_packet_info,

+ .hqd_reset = kgd_gfx_v9_hqd_reset,

+ .hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell

+void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,

+ uint32_t sch_wave,

+ uint32_t que_sleep,

+uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev,

+ int engine, int queue);

+ if (adev->apu_prefer_gtt) {

+ adev->apu_prefer_gtt ?

+ if (adev->apu_prefer_gtt) {

+/**

+ * amdgpu_amdkfd_remove_all_eviction_fences - Remove all eviction fences

+ * @bo: the BO where to remove the evictions fences from.

+ *

+ * This functions should only be used on release when all references to the BO

+ * are already dropped. We remove the eviction fence from the private copy of

+ * the dma_resv object here since that is what is used during release to

+ * determine of the BO is idle or not.

+ */

+void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo)

+ struct dma_resv *resv = &bo->tbo.base._resv;

+ struct dma_fence *fence, *stub;

+ struct dma_resv_iter cursor;

+ dma_resv_assert_held(resv);

+ stub = dma_fence_get_stub();

+ dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {

+ if (!to_amdgpu_amdkfd_fence(fence))

+ continue;

+ dma_resv_replace_fences(resv, fence->context, stub,

+ DMA_RESV_USAGE_BOOKKEEP);

+ }

+ dma_fence_put(stub);

+ return amdgpu_sync_fence(sync, vm->last_update, GFP_KERNEL);

+ if ((adev != bo_adev && !adev->apu_prefer_gtt) &&

+ (void)amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL);

+ return amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL);

+ if (adev->apu_prefer_gtt) {

+ if (adev->apu_prefer_gtt) {

+ (adev->apu_prefer_gtt &&

+ !adev->apu_prefer_gtt ?

+ ret = amdgpu_sync_fence(&sync_obj, fence, GFP_KERNEL);

+void amdgpu_atombios_oem_i2c_init(struct amdgpu_device *adev, u8 i2c_id)

+{

+ struct atom_context *ctx = adev->mode_info.atom_context;

+ ATOM_GPIO_I2C_ASSIGMENT *gpio;

+ struct amdgpu_i2c_bus_rec i2c;

+ int index = GetIndexIntoMasterTable(DATA, GPIO_I2C_Info);

+ struct _ATOM_GPIO_I2C_INFO *i2c_info;

+ uint16_t data_offset, size;

+ int i, num_indices;

+ char stmp[32];

+

+ if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) {

+ i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset);

+

+ num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) /

+ sizeof(ATOM_GPIO_I2C_ASSIGMENT);

+

+ gpio = &i2c_info->asGPIO_Info[0];

+ for (i = 0; i < num_indices; i++) {

+ i2c = amdgpu_atombios_get_bus_rec_for_i2c_gpio(gpio);

+

+ if (i2c.valid && i2c.i2c_id == i2c_id) {

+ sprintf(stmp, "OEM 0x%x", i2c.i2c_id);

+ adev->i2c_bus[i] = amdgpu_i2c_create(adev_to_drm(adev), &i2c, stmp);

+ break;

+ }

+ gpio = (ATOM_GPIO_I2C_ASSIGMENT *)

+ ((u8 *)gpio + sizeof(ATOM_GPIO_I2C_ASSIGMENT));

+ }

+ }

+}

+

+void amdgpu_atombios_oem_i2c_init(struct amdgpu_device *adev, u8 i2c_id);

+ bool mem_ecc_enabled = false;

+ adev->ras_default_ecc_enabled = false;

+ mem_ecc_enabled =

+ mem_ecc_enabled =

+ mem_ecc_enabled =

+ adev->ras_default_ecc_enabled =

+ (umc_config & UMC_CONFIG__DEFAULT_MEM_ECC_ENABLE) ? true : false;

+ umc_config = le32_to_cpu(umc_info->v40.umc_config);

+ mem_ecc_enabled =

+ adev->ras_default_ecc_enabled =

+ (umc_config & UMC_CONFIG__DEFAULT_MEM_ECC_ENABLE) ? true : false;

+ return mem_ecc_enabled;

+void amdgpu_bios_release(struct amdgpu_device *adev)

+{

+ kfree(adev->bios);

+ adev->bios = NULL;

+ adev->bios_size = 0;

+}

+

+ amdgpu_bios_release(adev);

+ amdgpu_bios_release(adev);

+ amdgpu_bios_release(adev);

+ amdgpu_bios_release(adev);

+

+ amdgpu_bios_release(adev);

+ amdgpu_bios_release(adev);

+ const struct drm_display_mode *mode)

+ const struct drm_display_mode *mode)

+ const struct drm_display_mode *mode)

+ const struct drm_display_mode *mode)

+// SPDX-License-Identifier: GPL-2.0

+/*

+ * Copyright 2025 Advanced Micro Devices, Inc.

+ *

+ * Permission is hereby granted, free of charge, to any person obtaining a

+ * copy of this software and associated documentation files (the "Software"),

+ * to deal in the Software without restriction, including without limitation

+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,

+ * and/or sell copies of the Software, and to permit persons to whom the

+ * Software is furnished to do so, subject to the following conditions:

+ *

+ * The above copyright notice and this permission notice shall be included in

+ * all copies or substantial portions of the Software.

+ *

+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL

+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR

+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,

+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR

+ * OTHER DEALINGS IN THE SOFTWARE.

+ *

+ */

+#include <linux/list.h>

+#include "amdgpu.h"

+

+static const guid_t MCE = CPER_NOTIFY_MCE;

+static const guid_t CMC = CPER_NOTIFY_CMC;

+static const guid_t BOOT = BOOT_TYPE;

+

+static const guid_t CRASHDUMP = AMD_CRASHDUMP;

+static const guid_t RUNTIME = AMD_GPU_NONSTANDARD_ERROR;

+

+static void __inc_entry_length(struct cper_hdr *hdr, uint32_t size)

+{

+ hdr->record_length += size;

+}

+

+static void amdgpu_cper_get_timestamp(struct cper_timestamp *timestamp)

+{

+ struct tm tm;

+ time64_t now = ktime_get_real_seconds();

+

+ time64_to_tm(now, 0, &tm);

+ timestamp->seconds = tm.tm_sec;

+ timestamp->minutes = tm.tm_min;

+ timestamp->hours = tm.tm_hour;

+ timestamp->flag = 0;

+ timestamp->day = tm.tm_mday;

+ timestamp->month = 1 + tm.tm_mon;

+ timestamp->year = (1900 + tm.tm_year) % 100;

+ timestamp->century = (1900 + tm.tm_year) / 100;

+}

+

+void amdgpu_cper_entry_fill_hdr(struct amdgpu_device *adev,

+ struct cper_hdr *hdr,

+ enum amdgpu_cper_type type,

+ enum cper_error_severity sev)

+{

+ char record_id[16];

+

+ hdr->signature[0] = 'C';

+ hdr->signature[1] = 'P';

+ hdr->signature[2] = 'E';

+ hdr->signature[3] = 'R';

+ hdr->revision = CPER_HDR_REV_1;

+ hdr->signature_end = 0xFFFFFFFF;

+ hdr->error_severity = sev;

+

+ hdr->valid_bits.platform_id = 1;

+ hdr->valid_bits.partition_id = 1;

+ hdr->valid_bits.timestamp = 1;

+

+ amdgpu_cper_get_timestamp(&hdr->timestamp);

+

+ snprintf(record_id, 9, "%d:%X",

+ (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) ?

+ adev->smuio.funcs->get_socket_id(adev) :

+ 0,

+ atomic_inc_return(&adev->cper.unique_id));

+ memcpy(hdr->record_id, record_id, 8);

+

+ snprintf(hdr->platform_id, 16, "0x%04X:0x%04X",

+ adev->pdev->vendor, adev->pdev->device);

+ /* pmfw version should be part of creator_id according to CPER spec */

+ snprintf(hdr->creator_id, 16, "%s", CPER_CREATOR_ID_AMDGPU);

+

+ switch (type) {

+ case AMDGPU_CPER_TYPE_BOOT:

+ hdr->notify_type = BOOT;

+ break;

+ case AMDGPU_CPER_TYPE_FATAL:

+ case AMDGPU_CPER_TYPE_BP_THRESHOLD:

+ hdr->notify_type = MCE;

+ break;

+ case AMDGPU_CPER_TYPE_RUNTIME:

+ if (sev == CPER_SEV_NON_FATAL_CORRECTED)

+ hdr->notify_type = CMC;

+ else

+ hdr->notify_type = MCE;

+ break;

+ default:

+ dev_err(adev->dev, "Unknown CPER Type\n");

+ break;

+ }

+

+ __inc_entry_length(hdr, HDR_LEN);

+}

+

+static int amdgpu_cper_entry_fill_section_desc(struct amdgpu_device *adev,

+ struct cper_sec_desc *section_desc,

+ bool bp_threshold,

+ bool poison,

+ enum cper_error_severity sev,

+ guid_t sec_type,

+ uint32_t section_length,

+ uint32_t section_offset)

+{

+ section_desc->revision_minor = CPER_SEC_MINOR_REV_1;

+ section_desc->revision_major = CPER_SEC_MAJOR_REV_22;

+ section_desc->sec_offset = section_offset;

+ section_desc->sec_length = section_length;

+ section_desc->valid_bits.fru_text = 1;

+ section_desc->flag_bits.primary = 1;

+ section_desc->severity = sev;

+ section_desc->sec_type = sec_type;

+

+ snprintf(section_desc->fru_text, 20, "OAM%d",

+ (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) ?

+ adev->smuio.funcs->get_socket_id(adev) :

+ 0);

+

+ if (bp_threshold)

+ section_desc->flag_bits.exceed_err_threshold = 1;

+ if (poison)

+ section_desc->flag_bits.latent_err = 1;

+

+ return 0;

+}

+

+int amdgpu_cper_entry_fill_fatal_section(struct amdgpu_device *adev,

+ struct cper_hdr *hdr,

+ uint32_t idx,

+ struct cper_sec_crashdump_reg_data reg_data)

+{

+ struct cper_sec_desc *section_desc;

+ struct cper_sec_crashdump_fatal *section;

+

+ section_desc = (struct cper_sec_desc *)((uint8_t *)hdr + SEC_DESC_OFFSET(idx));

+ section = (struct cper_sec_crashdump_fatal *)((uint8_t *)hdr +

+ FATAL_SEC_OFFSET(hdr->sec_cnt, idx));

+

+ amdgpu_cper_entry_fill_section_desc(adev, section_desc, false, false,

+ CPER_SEV_FATAL, CRASHDUMP, FATAL_SEC_LEN,

+ FATAL_SEC_OFFSET(hdr->sec_cnt, idx));

+

+ section->body.reg_ctx_type = CPER_CTX_TYPE_CRASH;

+ section->body.reg_arr_size = sizeof(reg_data);

+ section->body.data = reg_data;

+

+ __inc_entry_length(hdr, SEC_DESC_LEN + FATAL_SEC_LEN);

+

+ return 0;

+}

+

+int amdgpu_cper_entry_fill_runtime_section(struct amdgpu_device *adev,

+ struct cper_hdr *hdr,

+ uint32_t idx,

+ enum cper_error_severity sev,

+ uint32_t *reg_dump,

+ uint32_t reg_count)

+{

+ struct cper_sec_desc *section_desc;

+ struct cper_sec_nonstd_err *section;

+ bool poison;

+

+ poison = (sev == CPER_SEV_NON_FATAL_CORRECTED) ? false : true;

+ section_desc = (struct cper_sec_desc *)((uint8_t *)hdr + SEC_DESC_OFFSET(idx));

+ section = (struct cper_sec_nonstd_err *)((uint8_t *)hdr +

+ NONSTD_SEC_OFFSET(hdr->sec_cnt, idx));

+

+ amdgpu_cper_entry_fill_section_desc(adev, section_desc, false, poison,

+ sev, RUNTIME, NONSTD_SEC_LEN,

+ NONSTD_SEC_OFFSET(hdr->sec_cnt, idx));

+

+ reg_count = umin(reg_count, CPER_ACA_REG_COUNT);

+

+ section->hdr.valid_bits.err_info_cnt = 1;

+ section->hdr.valid_bits.err_context_cnt = 1;

+

+ section->info.error_type = RUNTIME;

+ section->info.ms_chk_bits.err_type_valid = 1;

+ section->ctx.reg_ctx_type = CPER_CTX_TYPE_CRASH;

+ section->ctx.reg_arr_size = sizeof(section->ctx.reg_dump);

+

+ memcpy(section->ctx.reg_dump, reg_dump, reg_count * sizeof(uint32_t));

+

+ __inc_entry_length(hdr, SEC_DESC_LEN + NONSTD_SEC_LEN);

+

+ return 0;

+}

+

+int amdgpu_cper_entry_fill_bad_page_threshold_section(struct amdgpu_device *adev,

+ struct cper_hdr *hdr,

+ uint32_t idx)

+{

+ struct cper_sec_desc *section_desc;

+ struct cper_sec_nonstd_err *section;

+

+ section_desc = (struct cper_sec_desc *)((uint8_t *)hdr + SEC_DESC_OFFSET(idx));

+ section = (struct cper_sec_nonstd_err *)((uint8_t *)hdr +

+ NONSTD_SEC_OFFSET(hdr->sec_cnt, idx));

+

+ amdgpu_cper_entry_fill_section_desc(adev, section_desc, true, false,

+ CPER_SEV_NUM, RUNTIME, NONSTD_SEC_LEN,

+ NONSTD_SEC_OFFSET(hdr->sec_cnt, idx));

+

+ section->hdr.valid_bits.err_info_cnt = 1;

+ section->hdr.valid_bits.err_context_cnt = 1;

+

+ section->info.error_type = RUNTIME;

+ section->info.ms_chk_bits.err_type_valid = 1;

+ section->ctx.reg_ctx_type = CPER_CTX_TYPE_CRASH;

+ section->ctx.reg_arr_size = sizeof(section->ctx.reg_dump);

+

+ /* Hardcoded Reg dump for bad page threshold CPER */

+ section->ctx.reg_dump[CPER_ACA_REG_CTL_LO] = 0x1;

+ section->ctx.reg_dump[CPER_ACA_REG_CTL_HI] = 0x0;

+ section->ctx.reg_dump[CPER_ACA_REG_STATUS_LO] = 0x137;

+ section->ctx.reg_dump[CPER_ACA_REG_STATUS_HI] = 0xB0000000;

+ section->ctx.reg_dump[CPER_ACA_REG_ADDR_LO] = 0x0;

+ section->ctx.reg_dump[CPER_ACA_REG_ADDR_HI] = 0x0;

+ section->ctx.reg_dump[CPER_ACA_REG_MISC0_LO] = 0x0;

+ section->ctx.reg_dump[CPER_ACA_REG_MISC0_HI] = 0x0;

+ section->ctx.reg_dump[CPER_ACA_REG_CONFIG_LO] = 0x2;

+ section->ctx.reg_dump[CPER_ACA_REG_CONFIG_HI] = 0x1ff;

+ section->ctx.reg_dump[CPER_ACA_REG_IPID_LO] = 0x0;

+ section->ctx.reg_dump[CPER_ACA_REG_IPID_HI] = 0x96;

+ section->ctx.reg_dump[CPER_ACA_REG_SYND_LO] = 0x0;

+ section->ctx.reg_dump[CPER_ACA_REG_SYND_HI] = 0x0;

+

+ __inc_entry_length(hdr, SEC_DESC_LEN + NONSTD_SEC_LEN);

+

+ return 0;

+}

+

+struct cper_hdr *amdgpu_cper_alloc_entry(struct amdgpu_device *adev,

+ enum amdgpu_cper_type type,

+ uint16_t section_count)

+{

+ struct cper_hdr *hdr;

+ uint32_t size = 0;

+

+ size += HDR_LEN;

+ size += (SEC_DESC_LEN * section_count);

+

+ switch (type) {

+ case AMDGPU_CPER_TYPE_RUNTIME:

+ case AMDGPU_CPER_TYPE_BP_THRESHOLD:

+ size += (NONSTD_SEC_LEN * section_count);

+ break;

+ case AMDGPU_CPER_TYPE_FATAL:

+ size += (FATAL_SEC_LEN * section_count);

+ break;

+ case AMDGPU_CPER_TYPE_BOOT:

+ size += (BOOT_SEC_LEN * section_count);

+ break;

+ default:

+ dev_err(adev->dev, "Unknown CPER Type!\n");

+ return NULL;

+ }

+

+ hdr = kzalloc(size, GFP_KERNEL);

+ if (!hdr)

+ return NULL;

+

+ /* Save this early */

+ hdr->sec_cnt = section_count;

+

+ return hdr;

+}

+

+int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev,

+ struct aca_bank *bank)

+{

+ struct cper_hdr *fatal = NULL;

+ struct cper_sec_crashdump_reg_data reg_data = { 0 };

+ struct amdgpu_ring *ring = &adev->cper.ring_buf;

+ int ret;

+

+ fatal = amdgpu_cper_alloc_entry(adev, AMDGPU_CPER_TYPE_FATAL, 1);

+ if (!fatal) {

+ dev_err(adev->dev, "fail to alloc cper entry for ue record\n");

+ return -ENOMEM;

+ }

+

+ reg_data.status_lo = lower_32_bits(bank->regs[ACA_REG_IDX_STATUS]);

+ reg_data.status_hi = upper_32_bits(bank->regs[ACA_REG_IDX_STATUS]);

+ reg_data.addr_lo = lower_32_bits(bank->regs[ACA_REG_IDX_ADDR]);

+ reg_data.addr_hi = upper_32_bits(bank->regs[ACA_REG_IDX_ADDR]);

+ reg_data.ipid_lo = lower_32_bits(bank->regs[ACA_REG_IDX_IPID]);

+ reg_data.ipid_hi = upper_32_bits(bank->regs[ACA_REG_IDX_IPID]);

+ reg_data.synd_lo = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]);

+ reg_data.synd_hi = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]);

+

+ amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL);

+ ret = amdgpu_cper_entry_fill_fatal_section(adev, fatal, 0, reg_data);

+ if (ret)

+ return ret;

+

+ amdgpu_cper_ring_write(ring, fatal, fatal->record_length);

+ kfree(fatal);

+

+ return 0;

+}

+

+int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev)

+{

+ struct cper_hdr *bp_threshold = NULL;

+ struct amdgpu_ring *ring = &adev->cper.ring_buf;

+ int ret;

+

+ bp_threshold = amdgpu_cper_alloc_entry(adev, AMDGPU_CPER_TYPE_BP_THRESHOLD, 1);

+ if (!bp_threshold) {

+ dev_err(adev->dev, "fail to alloc cper entry for bad page threshold record\n");

+ return -ENOMEM;

+ }

+

+ amdgpu_cper_entry_fill_hdr(adev, bp_threshold, AMDGPU_CPER_TYPE_BP_THRESHOLD, CPER_SEV_NUM);

+ ret = amdgpu_cper_entry_fill_bad_page_threshold_section(adev, bp_threshold, 0);

+ if (ret)

+ return ret;

+

+ amdgpu_cper_ring_write(ring, bp_threshold, bp_threshold->record_length);

+ kfree(bp_threshold);

+

+ return 0;

+}

+

+static enum cper_error_severity amdgpu_aca_err_type_to_cper_sev(struct amdgpu_device *adev,

+ enum aca_error_type aca_err_type)

+{

+ switch (aca_err_type) {

+ case ACA_ERROR_TYPE_UE:

+ return CPER_SEV_FATAL;

+ case ACA_ERROR_TYPE_CE:

+ return CPER_SEV_NON_FATAL_CORRECTED;

+ case ACA_ERROR_TYPE_DEFERRED:

+ return CPER_SEV_NON_FATAL_UNCORRECTED;

+ default:

+ dev_err(adev->dev, "Unknown ACA error type!\n");

+ return CPER_SEV_FATAL;

+ }

+}

+

+int amdgpu_cper_generate_ce_records(struct amdgpu_device *adev,

+ struct aca_banks *banks,

+ uint16_t bank_count)

+{

+ struct cper_hdr *corrected = NULL;

+ enum cper_error_severity sev = CPER_SEV_NON_FATAL_CORRECTED;

+ struct amdgpu_ring *ring = &adev->cper.ring_buf;

+ uint32_t reg_data[CPER_ACA_REG_COUNT] = { 0 };

+ struct aca_bank_node *node;

+ struct aca_bank *bank;

+ uint32_t i = 0;

+ int ret;

+

+ corrected = amdgpu_cper_alloc_entry(adev, AMDGPU_CPER_TYPE_RUNTIME, bank_count);

+ if (!corrected) {

+ dev_err(adev->dev, "fail to allocate cper entry for ce records\n");

+ return -ENOMEM;

+ }

+

+ /* Raise severity if any DE is detected in the ACA bank list */

+ list_for_each_entry(node, &banks->list, node) {

+ bank = &node->bank;

+ if (bank->aca_err_type == ACA_ERROR_TYPE_DEFERRED) {

+ sev = CPER_SEV_NON_FATAL_UNCORRECTED;

+ break;

+ }

+ }

+

+ amdgpu_cper_entry_fill_hdr(adev, corrected, AMDGPU_CPER_TYPE_RUNTIME, sev);

+

+ /* Combine CE and DE in cper record */

+ list_for_each_entry(node, &banks->list, node) {

+ bank = &node->bank;

+ reg_data[CPER_ACA_REG_CTL_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_CTL]);

+ reg_data[CPER_ACA_REG_CTL_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_CTL]);

+ reg_data[CPER_ACA_REG_STATUS_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_STATUS]);

+ reg_data[CPER_ACA_REG_STATUS_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_STATUS]);

+ reg_data[CPER_ACA_REG_ADDR_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_ADDR]);

+ reg_data[CPER_ACA_REG_ADDR_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_ADDR]);

+ reg_data[CPER_ACA_REG_MISC0_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_MISC0]);

+ reg_data[CPER_ACA_REG_MISC0_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_MISC0]);

+ reg_data[CPER_ACA_REG_CONFIG_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_CONFIG]);

+ reg_data[CPER_ACA_REG_CONFIG_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_CONFIG]);

+ reg_data[CPER_ACA_REG_IPID_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_IPID]);

+ reg_data[CPER_ACA_REG_IPID_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_IPID]);

+ reg_data[CPER_ACA_REG_SYND_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]);

+ reg_data[CPER_ACA_REG_SYND_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]);

+

+ ret = amdgpu_cper_entry_fill_runtime_section(adev, corrected, i++,

+ amdgpu_aca_err_type_to_cper_sev(adev, bank->aca_err_type),

+ reg_data, CPER_ACA_REG_COUNT);

+ if (ret)

+ return ret;

+ }

+

+ amdgpu_cper_ring_write(ring, corrected, corrected->record_length);

+ kfree(corrected);

+

+ return 0;

+}

+

+static bool amdgpu_cper_is_hdr(struct amdgpu_ring *ring, u64 pos)

+{

+ struct cper_hdr *chdr;

+

+ chdr = (struct cper_hdr *)&(ring->ring[pos]);

+ return strcmp(chdr->signature, "CPER") ? false : true;

+}

+

+static u32 amdgpu_cper_ring_get_ent_sz(struct amdgpu_ring *ring, u64 pos)

+{

+ struct cper_hdr *chdr;

+ u64 p;

+ u32 chunk, rec_len = 0;

+

+ chdr = (struct cper_hdr *)&(ring->ring[pos]);

+ chunk = ring->ring_size - (pos << 2);

+

+ if (!strcmp(chdr->signature, "CPER")) {

+ rec_len = chdr->record_length;

+ goto calc;

+ }

+

+ /* ring buffer is not full, no cper data after ring->wptr */

+ if (ring->count_dw)

+ goto calc;

+

+ for (p = pos + 1; p <= ring->buf_mask; p++) {

+ chdr = (struct cper_hdr *)&(ring->ring[p]);

+ if (!strcmp(chdr->signature, "CPER")) {

+ rec_len = (p - pos) << 2;

+ goto calc;

+ }

+ }

+

+calc:

+ if (!rec_len)

+ return chunk;

+ else

+ return umin(rec_len, chunk);

+}

+

+void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count)

+{

+ u64 pos, wptr_old, rptr = *ring->rptr_cpu_addr & ring->ptr_mask;

+ int rec_cnt_dw = count >> 2;

+ u32 chunk, ent_sz;

+ u8 *s = (u8 *)src;

+

+ if (count >= ring->ring_size - 4) {

+ dev_err(ring->adev->dev,

+ "CPER data size(%d) is larger than ring size(%d)\n",

+ count, ring->ring_size - 4);

+

+ return;

+ }

+

+ wptr_old = ring->wptr;

+

+ mutex_lock(&ring->adev->cper.ring_lock);

+ while (count) {

+ ent_sz = amdgpu_cper_ring_get_ent_sz(ring, ring->wptr);

+ chunk = umin(ent_sz, count);

+

+ memcpy(&ring->ring[ring->wptr], s, chunk);

+

+ ring->wptr += (chunk >> 2);

+ ring->wptr &= ring->ptr_mask;

+ count -= chunk;

+ s += chunk;

+ }

+

+ if (ring->count_dw < rec_cnt_dw)

+ ring->count_dw = 0;

+

+ /* the buffer is overflow, adjust rptr */

+ if (((wptr_old < rptr) && (rptr <= ring->wptr)) ||

+ ((ring->wptr < wptr_old) && (wptr_old < rptr)) ||

+ ((rptr <= ring->wptr) && (ring->wptr < wptr_old))) {

+ pos = (ring->wptr + 1) & ring->ptr_mask;

+

+ do {

+ ent_sz = amdgpu_cper_ring_get_ent_sz(ring, pos);

+

+ rptr += (ent_sz >> 2);

+ rptr &= ring->ptr_mask;

+ *ring->rptr_cpu_addr = rptr;

+

+ pos = rptr;

+ } while (!amdgpu_cper_is_hdr(ring, rptr));

+ }

+

+ if (ring->count_dw >= rec_cnt_dw)

+ ring->count_dw -= rec_cnt_dw;

+ mutex_unlock(&ring->adev->cper.ring_lock);

+}

+

+static u64 amdgpu_cper_ring_get_rptr(struct amdgpu_ring *ring)

+{

+ return *(ring->rptr_cpu_addr);

+}

+

+static u64 amdgpu_cper_ring_get_wptr(struct amdgpu_ring *ring)

+{

+ return ring->wptr;

+}

+

+static const struct amdgpu_ring_funcs cper_ring_funcs = {

+ .type = AMDGPU_RING_TYPE_CPER,

+ .align_mask = 0xff,

+ .support_64bit_ptrs = false,

+ .get_rptr = amdgpu_cper_ring_get_rptr,

+ .get_wptr = amdgpu_cper_ring_get_wptr,

+};

+

+static int amdgpu_cper_ring_init(struct amdgpu_device *adev)

+{

+ struct amdgpu_ring *ring = &(adev->cper.ring_buf);

+

+ mutex_init(&adev->cper.ring_lock);

+

+ ring->adev = NULL;

+ ring->ring_obj = NULL;

+ ring->use_doorbell = false;

+ ring->no_scheduler = true;

+ ring->funcs = &cper_ring_funcs;

+

+ sprintf(ring->name, "cper");

+ return amdgpu_ring_init(adev, ring, CPER_MAX_RING_SIZE, NULL, 0,

+ AMDGPU_RING_PRIO_DEFAULT, NULL);

+}

+

+int amdgpu_cper_init(struct amdgpu_device *adev)

+{

+ int r;

+

+ if (!amdgpu_aca_is_enabled(adev))

+ return 0;

+

+ r = amdgpu_cper_ring_init(adev);

+ if (r) {

+ dev_err(adev->dev, "failed to initialize cper ring, r = %d\n", r);

+ return r;

+ }

+

+ mutex_init(&adev->cper.cper_lock);

+

+ adev->cper.enabled = true;

+ adev->cper.max_count = CPER_MAX_ALLOWED_COUNT;

+

+ return 0;

+}

+

+int amdgpu_cper_fini(struct amdgpu_device *adev)

+{

+ if (!amdgpu_aca_is_enabled(adev))

+ return 0;

+

+ adev->cper.enabled = false;

+

+ amdgpu_ring_fini(&(adev->cper.ring_buf));

+ adev->cper.count = 0;

+ adev->cper.wptr = 0;

+

+ return 0;

+}

+/* SPDX-License-Identifier: GPL-2.0 */

+/*

+ * Copyright 2025 Advanced Micro Devices, Inc.

+ *

+ * Permission is hereby granted, free of charge, to any person obtaining a

+ * copy of this software and associated documentation files (the "Software"),

+ * to deal in the Software without restriction, including without limitation

+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,

+ * and/or sell copies of the Software, and to permit persons to whom the

+ * Software is furnished to do so, subject to the following conditions:

+ *

+ * The above copyright notice and this permission notice shall be included in

+ * all copies or substantial portions of the Software.

+ *

+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL

+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR

+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,

+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR

+ * OTHER DEALINGS IN THE SOFTWARE.

+ *

+ */

+

+#ifndef __AMDGPU_CPER_H__

+#define __AMDGPU_CPER_H__

+

+#include "amd_cper.h"

+#include "amdgpu_aca.h"

+

+#define CPER_MAX_ALLOWED_COUNT 0x1000

+#define CPER_MAX_RING_SIZE 0X100000

+#define HDR_LEN (sizeof(struct cper_hdr))

+#define SEC_DESC_LEN (sizeof(struct cper_sec_desc))

+

+#define BOOT_SEC_LEN (sizeof(struct cper_sec_crashdump_boot))

+#define FATAL_SEC_LEN (sizeof(struct cper_sec_crashdump_fatal))

+#define NONSTD_SEC_LEN (sizeof(struct cper_sec_nonstd_err))

+

+#define SEC_DESC_OFFSET(idx) (HDR_LEN + (SEC_DESC_LEN * idx))

+

+#define BOOT_SEC_OFFSET(count, idx) (HDR_LEN + (SEC_DESC_LEN * count) + (BOOT_SEC_LEN * idx))

+#define FATAL_SEC_OFFSET(count, idx) (HDR_LEN + (SEC_DESC_LEN * count) + (FATAL_SEC_LEN * idx))

+#define NONSTD_SEC_OFFSET(count, idx) (HDR_LEN + (SEC_DESC_LEN * count) + (NONSTD_SEC_LEN * idx))

+

+enum amdgpu_cper_type {

+ AMDGPU_CPER_TYPE_RUNTIME,

+ AMDGPU_CPER_TYPE_FATAL,

+ AMDGPU_CPER_TYPE_BOOT,

+ AMDGPU_CPER_TYPE_BP_THRESHOLD,

+};

+

+struct amdgpu_cper {

+ bool enabled;

+

+ atomic_t unique_id;

+ struct mutex cper_lock;

+

+ /* Lifetime CPERs generated */

+ uint32_t count;

+ uint32_t max_count;

+

+ uint32_t wptr;

+

+ void *ring[CPER_MAX_ALLOWED_COUNT];

+ struct amdgpu_ring ring_buf;

+ struct mutex ring_lock;

+};

+

+void amdgpu_cper_entry_fill_hdr(struct amdgpu_device *adev,

+ struct cper_hdr *hdr,

+ enum amdgpu_cper_type type,

+ enum cper_error_severity sev);

+int amdgpu_cper_entry_fill_fatal_section(struct amdgpu_device *adev,

+ struct cper_hdr *hdr,

+ uint32_t idx,

+ struct cper_sec_crashdump_reg_data reg_data);

+int amdgpu_cper_entry_fill_runtime_section(struct amdgpu_device *adev,

+ struct cper_hdr *hdr,

+ uint32_t idx,

+ enum cper_error_severity sev,

+ uint32_t *reg_dump,

+ uint32_t reg_count);

+int amdgpu_cper_entry_fill_bad_page_threshold_section(struct amdgpu_device *adev,

+ struct cper_hdr *hdr,

+ uint32_t section_idx);

+

+struct cper_hdr *amdgpu_cper_alloc_entry(struct amdgpu_device *adev,

+ enum amdgpu_cper_type type,

+ uint16_t section_count);

+/* UE must be encoded into separated cper entries, 1 UE 1 cper */

+int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev,

+ struct aca_bank *bank);

+/* CEs and DEs are combined into 1 cper entry */

+int amdgpu_cper_generate_ce_records(struct amdgpu_device *adev,

+ struct aca_banks *banks,

+ uint16_t bank_count);

+/* Bad page threshold is encoded into separated cper entry */

+int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev);

+void amdgpu_cper_ring_write(struct amdgpu_ring *ring,

+ void *src, int count);

+int amdgpu_cper_init(struct amdgpu_device *adev);

+int amdgpu_cper_fini(struct amdgpu_device *adev);

+

+#endif

+ r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL);

+ r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL);

+ if (amdgpu_vmid_uses_reserved(vm, ring->vm_hub))

+ r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update,

+ GFP_KERNEL);

+ r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,

+ GFP_KERNEL);

+ r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,

+ GFP_KERNEL);

+ r = amdgpu_sync_fence(&p->sync, vm->last_update, GFP_KERNEL);

+ r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence,

+ GFP_KERNEL);

+ if (amdgpu_sriov_multi_vf_mode(adev))

+

+ drm_info(dev, "AMDGPU device coredump file has been created\n");

+ drm_info(dev, "Check your /sys/class/drm/card%d/device/devcoredump/data\n",

+ dev->primary->index);

+#define AMDGPU_VBIOS_SKIP (1U << 0)

+#define AMDGPU_VBIOS_OPTIONAL (1U << 1)

+

+static int amdgpu_device_attr_sysfs_init(struct amdgpu_device *adev)

+{

+ int ret = 0;

+

+ if (!amdgpu_sriov_vf(adev))

+ ret = sysfs_create_file(&adev->dev->kobj,

+ &dev_attr_pcie_replay_count.attr);

+

+ return ret;

+}

+

+static void amdgpu_device_attr_sysfs_fini(struct amdgpu_device *adev)

+{

+ if (!amdgpu_sriov_vf(adev))

+ sysfs_remove_file(&adev->dev->kobj,

+ &dev_attr_pcie_replay_count.attr);

+}

+

+static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev)

+{

+ if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))

+ return AMDGPU_VBIOS_SKIP;

+

+ if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev))

+ return AMDGPU_VBIOS_OPTIONAL;

+

+ return 0;

+}

+

+ uint32_t flags;

+ bool optional;

+ flags = amdgpu_device_get_vbios_flags(adev);

+ optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP));

+ if (optional && !adev->bios)

+ return 0;

+

+ if (optional && !adev->bios)

+ return 0;

+

+ /* resizing on Dell G5 SE platforms causes problems with runtime pm */

+ if ((amdgpu_runtime_pm != 0) &&

+ adev->pdev->vendor == PCI_VENDOR_ID_ATI &&

+ adev->pdev->device == 0x731f &&

+ adev->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)

+ return 0;

+

+ uint32_t reg, flags;

+ flags = amdgpu_device_get_vbios_flags(adev);

+ if (flags & AMDGPU_VBIOS_SKIP)

+ return false;

+ if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios)

+ adev->ip_blocks[i].version->funcs->get_clockgating_state(

+ &adev->ip_blocks[i], flags);

+ bool total, skip_bios;

+ uint32_t bios_flags;

+ bios_flags = amdgpu_device_get_vbios_flags(adev);

+ skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP);

+ if (!skip_bios) {

+ bool optional =

+ !!(bios_flags & AMDGPU_VBIOS_OPTIONAL);

+ if (!amdgpu_get_bios(adev) && !optional)

+ if (optional && !adev->bios)

+ dev_info(

+ adev->dev,

+ "VBIOS image optional, proceeding without VBIOS image");

+

+ if (adev->bios) {

+ r = amdgpu_atombios_init(adev);

+ if (r) {

+ dev_err(adev->dev,

+ "amdgpu_atombios_init failed\n");

+ amdgpu_vf_error_put(

+ adev,

+ AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL,

+ 0, 0);

+ return r;

+ }

+ if (adev->gmc.xgmi.supported)

+ amdgpu_xgmi_early_init(adev);

+

+ struct drm_sched_init_args args = {

+ .ops = &amdgpu_sched_ops,

+ .num_rqs = DRM_SCHED_PRIORITY_COUNT,

+ .timeout_wq = adev->reset_domain->wq,

+ .dev = adev->dev,

+ };

+ args.timeout = timeout;

+ args.credit_limit = ring->num_hw_submission;

+ args.score = ring->sched_score;

+ args.name = ring->name;

+

+ r = drm_sched_init(&ring->sched, &args);

+ if (!amdgpu_sriov_vf(adev) || amdgpu_sriov_ras_cper_en(adev))

+ r = amdgpu_cper_init(adev);

+

+ amdgpu_cper_fini(adev);

+

+ for (i = 0; i < MAX_XCP; ++i) {

+ adev->isolation[i].spearhead = dma_fence_get_stub();

+ amdgpu_sync_create(&adev->isolation[i].active);

+ amdgpu_sync_create(&adev->isolation[i].prev);

+ }

+ mutex_init(&adev->gfx.workload_profile_mutex);

+ mutex_init(&adev->vcn.workload_profile_mutex);

+ spin_lock_init(&adev->virt.rlcg_reg_lock);

+ amdgpu_virt_init(adev);

+ /*

+ * No need to remove conflicting FBs for non-display class devices.

+ * This prevents the sysfb from being freed accidently.

+ */

+ if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA ||

+ (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) {

+ /* Get rid of things like offb */

+ r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name);

+ if (r)

+ return r;

+ }

+ amdgpu_i2c_init(adev);

+ r = amdgpu_device_attr_sysfs_init(adev);

+ amdgpu_device_attr_sysfs_fini(adev);

+ int i, idx;

+ for (i = 0; i < MAX_XCP; ++i) {

+ dma_fence_put(adev->isolation[i].spearhead);

+ amdgpu_sync_free(&adev->isolation[i].active);

+ amdgpu_sync_free(&adev->isolation[i].prev);

+ }

+ amdgpu_i2c_fini(adev);

+ if (adev->bios) {

+ if (amdgpu_emu_mode != 1)

+ amdgpu_atombios_fini(adev);

+ amdgpu_bios_release(adev);

+ }

+ kfree(adev->xcp_mgr);

+ adev->xcp_mgr = NULL;

+

+ amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||

+ if (adev->bios)

+ amdgpu_atombios_scratch_regs_engine_hung(adev, true);

+ if (adev->bios)

+ amdgpu_atombios_scratch_regs_engine_hung(adev, false);

+

+ if (!r)

+ drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE);

+

+ dma_fence_get(gang);

+ if (!dma_fence_is_signaled(old)) {

+ dma_fence_put(gang);

+ }

+ /*

+ * Drop it once for the exchanged reference in adev and once for the

+ * thread local reference acquired in amdgpu_device_get_gang().

+ */

+ dma_fence_put(old);

+/**

+ * amdgpu_device_enforce_isolation - enforce HW isolation

+ * @adev: the amdgpu device pointer

+ * @ring: the HW ring the job is supposed to run on

+ * @job: the job which is about to be pushed to the HW ring

+ *

+ * Makes sure that only one client at a time can use the GFX block.

+ * Returns: The dependency to wait on before the job can be pushed to the HW.

+ * The function is called multiple times until NULL is returned.

+ */

+struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,

+ struct amdgpu_ring *ring,

+ struct amdgpu_job *job)

+{

+ struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id];

+ struct drm_sched_fence *f = job->base.s_fence;

+ struct dma_fence *dep;

+ void *owner;

+ int r;

+

+ /*

+ * For now enforce isolation only for the GFX block since we only need

+ * the cleaner shader on those rings.

+ */

+ if (ring->funcs->type != AMDGPU_RING_TYPE_GFX &&

+ ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)

+ return NULL;

+

+ /*

+ * All submissions where enforce isolation is false are handled as if

+ * they come from a single client. Use ~0l as the owner to distinct it

+ * from kernel submissions where the owner is NULL.

+ */

+ owner = job->enforce_isolation ? f->owner : (void *)~0l;

+

+ mutex_lock(&adev->enforce_isolation_mutex);

+

+ /*

+ * The "spearhead" submission is the first one which changes the

+ * ownership to its client. We always need to wait for it to be

+ * pushed to the HW before proceeding with anything.

+ */

+ if (&f->scheduled != isolation->spearhead &&

+ !dma_fence_is_signaled(isolation->spearhead)) {

+ dep = isolation->spearhead;

+ goto out_grab_ref;

+ }

+

+ if (isolation->owner != owner) {

+

+ /*

+ * Wait for any gang to be assembled before switching to a

+ * different owner or otherwise we could deadlock the

+ * submissions.

+ */

+ if (!job->gang_submit) {

+ dep = amdgpu_device_get_gang(adev);

+ if (!dma_fence_is_signaled(dep))

+ goto out_return_dep;

+ dma_fence_put(dep);

+ }

+

+ dma_fence_put(isolation->spearhead);

+ isolation->spearhead = dma_fence_get(&f->scheduled);

+ amdgpu_sync_move(&isolation->active, &isolation->prev);

+ trace_amdgpu_isolation(isolation->owner, owner);

+ isolation->owner = owner;

+ }

+

+ /*

+ * Specifying the ring here helps to pipeline submissions even when

+ * isolation is enabled. If that is not desired for testing NULL can be

+ * used instead of the ring to enforce a CPU round trip while switching

+ * between clients.

+ */

+ dep = amdgpu_sync_peek_fence(&isolation->prev, ring);

+ r = amdgpu_sync_fence(&isolation->active, &f->finished, GFP_NOWAIT);

+ if (r)

+ DRM_WARN("OOM tracking isolation\n");

+

+out_grab_ref:

+ dma_fence_get(dep);

+out_return_dep:

+ mutex_unlock(&adev->enforce_isolation_mutex);

+ return dep;

+}

+

+MODULE_FIRMWARE("amdgpu/ip_discovery.bin");

+MODULE_FIRMWARE("amdgpu/vega10_ip_discovery.bin");

+MODULE_FIRMWARE("amdgpu/vega12_ip_discovery.bin");

+MODULE_FIRMWARE("amdgpu/vega20_ip_discovery.bin");

+MODULE_FIRMWARE("amdgpu/raven_ip_discovery.bin");

+MODULE_FIRMWARE("amdgpu/raven2_ip_discovery.bin");

+MODULE_FIRMWARE("amdgpu/picasso_ip_discovery.bin");

+static int amdgpu_discovery_read_binary_from_file(struct amdgpu_device *adev,

+ uint8_t *binary,

+ const char *fw_name)

+static const char *amdgpu_discovery_get_fw_name(struct amdgpu_device *adev)

+{

+ if (amdgpu_discovery == 2)

+ return "amdgpu/ip_discovery.bin";

+

+ switch (adev->asic_type) {

+ case CHIP_VEGA10:

+ return "amdgpu/vega10_ip_discovery.bin";

+ case CHIP_VEGA12:

+ return "amdgpu/vega12_ip_discovery.bin";

+ case CHIP_RAVEN:

+ if (adev->apu_flags & AMD_APU_IS_RAVEN2)

+ return "amdgpu/raven2_ip_discovery.bin";

+ else if (adev->apu_flags & AMD_APU_IS_PICASSO)

+ return "amdgpu/picasso_ip_discovery.bin";

+ else

+ return "amdgpu/raven_ip_discovery.bin";

+ case CHIP_VEGA20:

+ return "amdgpu/vega20_ip_discovery.bin";

+ case CHIP_ARCTURUS:

+ return "amdgpu/arcturus_ip_discovery.bin";

+ case CHIP_ALDEBARAN:

+ return "amdgpu/aldebaran_ip_discovery.bin";

+ default:

+ return NULL;

+ }

+}

+

+ const char *fw_name;

+ fw_name = amdgpu_discovery_get_fw_name(adev);

+ if (fw_name != NULL) {

+ r = amdgpu_discovery_read_binary_from_file(adev, adev->mman.discovery_bin, fw_name);

+static int amdgpu_discovery_validate_ip(struct amdgpu_device *adev,

+ uint8_t instance, uint16_t hw_id)

+ if (instance >= HWIP_MAX_INSTANCE) {

+ dev_err(adev->dev,

+ "Unexpected instance_number (%d) from ip discovery blob\n",

+ instance);

+ if (hw_id >= HW_ID_MAX) {

+ dev_err(adev->dev,

+ "Unexpected hw_id (%d) from ip discovery blob\n",

+ hw_id);

+ struct ip *ip;

+ uint16_t hw_id;

+ uint8_t inst;

+ ip = (struct ip *)(adev->mman.discovery_bin +

+ ip_offset);

+ inst = ip->number_instance;

+ hw_id = le16_to_cpu(ip->hw_id);

+ if (amdgpu_discovery_validate_ip(adev, inst, hw_id))

+ if (ip->harvest == 1) {

+ switch (hw_id) {

+ if (inst == 0) {

+ ip_offset += struct_size(ip, base_address,

+ ip->num_base_address);

+ uint16_t hw_id;

+ uint8_t inst;

+ inst = ip->instance_number;

+ hw_id = le16_to_cpu(ip->hw_id);

+ if (amdgpu_discovery_validate_ip(adev, inst, hw_id) ||

+ hw_id != ii)

+ uint32_t wafl_ver;

+ uint16_t hw_id;

+ uint8_t inst;

+ wafl_ver = 0;

+ inst = ip->instance_number;

+ hw_id = le16_to_cpu(ip->hw_id);

+ if (amdgpu_discovery_validate_ip(adev, inst, hw_id))

+ if (!wafl_ver && le16_to_cpu(ip->hw_id) == WAFLC_HWID)

+ wafl_ver = IP_VERSION_FULL(ip->major, ip->minor,

+ ip->revision, 0, 0);

+

+ if (wafl_ver && !adev->ip_versions[XGMI_HWIP][0])

+ adev->ip_versions[XGMI_HWIP][0] = wafl_ver;

+

+ struct ip_discovery_header *ihdr;

+ struct binary_header *bhdr;

+ uint16_t offset, ihdr_ver;

+ bhdr = (struct binary_header *)adev->mman.discovery_bin;

+ offset = le16_to_cpu(bhdr->table_list[IP_DISCOVERY].offset);

+ ihdr = (struct ip_discovery_header *)(adev->mman.discovery_bin +

+ offset);

+ ihdr_ver = le16_to_cpu(ihdr->version);

+ ihdr_ver <= 2) {

+ case IP_VERSION(11, 5, 3):

+ case IP_VERSION(11, 5, 3):

+ case IP_VERSION(11, 5, 2):

+ case IP_VERSION(14, 0, 5):

+ case IP_VERSION(11, 5, 2):

+ case IP_VERSION(13, 0, 12):

+ case IP_VERSION(14, 0, 5):

+ case IP_VERSION(3, 6, 0):

+ case IP_VERSION(11, 5, 3):

+ case IP_VERSION(6, 1, 3):

+ case IP_VERSION(11, 5, 3):

+ case CHIP_VEGA12:

+ case CHIP_RAVEN:

+ case CHIP_VEGA20:

+ case CHIP_ARCTURUS:

+ case CHIP_ALDEBARAN:

+ /* this is not fatal. We have a fallback below

+ * if the new firmwares are not present. some of

+ * this will be overridden below to keep things

+ * consistent with the current behavior.

+ */

+ r = amdgpu_discovery_reg_base_init(adev);

+ if (!r) {

+ amdgpu_discovery_harvest_ip(adev);

+ amdgpu_discovery_get_gfx_info(adev);

+ amdgpu_discovery_get_mall_info(adev);

+ amdgpu_discovery_get_vcn_info(adev);

+ }

+ break;

+ default:

+ r = amdgpu_discovery_reg_base_init(adev);

+ if (r)

+ return -EINVAL;

+

+ amdgpu_discovery_harvest_ip(adev);

+ amdgpu_discovery_get_gfx_info(adev);

+ amdgpu_discovery_get_mall_info(adev);

+ amdgpu_discovery_get_vcn_info(adev);

+ break;

+ }

+

+ switch (adev->asic_type) {

+ case CHIP_VEGA10:

+ case IP_VERSION(11, 5, 3):

+ case IP_VERSION(11, 5, 3):

+ case IP_VERSION(7, 9, 1):

+ case IP_VERSION(7, 11, 2):

+ case IP_VERSION(11, 5, 2):

+ case IP_VERSION(13, 0, 11):

+ * - 3.60.0 - Add AMDGPU_TILING_GFX12_DCC_WRITE_COMPRESS_DISABLE (Vulkan requirement)

+ * - 3.61.0 - Contains fix for RV/PCO compute queues

+ * - 3.62.0 - Add AMDGPU_IDS_FLAGS_MODE_PF, AMDGPU_IDS_FLAGS_MODE_VF & AMDGPU_IDS_FLAGS_MODE_PT

+ * - 3.63.0 - GFX12 display DCC supports 256B max compressed block size

+#define KMS_DRIVER_MINOR 63

+ AMDGPU_DEBUG_DISABLE_GPU_RING_RESET = BIT(6),

+ AMDGPU_DEBUG_SMU_POOL = BIT(7),

+int amdgpu_modeset = -1;

+ * This parameter is deprecated and will be removed in the future.

+MODULE_PARM_DESC(bad_page_threshold, "Bad page threshold(-1 = ignore threshold (default value), 0 = disable bad page retirement, -2 = threshold determined by a formula, 0 < threshold < max records, user-defined threshold)");

+ * DOC: modeset (int)

+ * Override nomodeset (1 = override, -1 = auto). The default is -1 (auto).

+ */

+MODULE_PARM_DESC(modeset, "Override nomodeset (1 = enable, -1 = auto)");

+module_param_named(modeset, amdgpu_modeset, int, 0444);

+

+/**

+ * - 0x8: Use VRAM for firmware loading

+ * - 0x10: Enable ACA based RAS logging

+ * - 0x20: Enable experimental resets

+ * - 0x40: Disable ring resets

+ * - 0x80: Use VRAM for SMU pool

+

+ if (amdgpu_debug_mask & AMDGPU_DEBUG_DISABLE_GPU_RING_RESET) {

+ pr_info("debug: ring reset disabled\n");

+ adev->debug_disable_gpu_ring_reset = true;

+ }

+ if (amdgpu_debug_mask & AMDGPU_DEBUG_SMU_POOL) {

+ pr_info("debug: use vram for smu pool\n");

+ adev->pm.smu_debug_mask |= SMU_DEBUG_POOL_USE_VRAM;

+ }

+ if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA ||

+ (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) {

+ if (drm_firmware_drivers_only() && amdgpu_modeset == -1)

+ return -EINVAL;

+ }

+

+ struct amdgpu_device *adev = drm_to_adev(drm_dev);

+ int r;

+ r = amdgpu_device_resume(drm_dev, true);

+ adev->in_s4 = false;

+

+ return r;

+ struct amdgpu_device *adev = drm_to_adev(drm_dev);

+

+ adev->in_s4 = false;

+#define FRU_EEPROM_MADDR_INV 0xFFFFF

+ if (atom_ctx && (strnstr(atom_ctx->vbios_pn, "D161",

+ sizeof(atom_ctx->vbios_pn)) ||

+ strnstr(atom_ctx->vbios_pn, "D163",

+ sizeof(atom_ctx->vbios_pn)))) {

+ if (atom_ctx && strnstr(atom_ctx->vbios_pn, "D603",

+ sizeof(atom_ctx->vbios_pn))) {

+ if (atom_ctx && !strnstr(atom_ctx->vbios_pn, "D673",

+ sizeof(atom_ctx->vbios_pn)))

+ case IP_VERSION(13, 0, 12):

+ if (fru_addr)

+ *fru_addr = FRU_EEPROM_MADDR_INV;

+ return true;

+ /* FRU data avaialble, but no direct EEPROM access */

+ if (fru_addr == FRU_EEPROM_MADDR_INV)

+ return 0;

+

+static void amdgpu_gfx_do_off_ctrl(struct amdgpu_device *adev, bool enable,

+ bool no_delay)

+ if (no_delay) {

+/* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable

+ *

+ * @adev: amdgpu_device pointer

+ * @bool enable true: enable gfx off feature, false: disable gfx off feature

+ *

+ * 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled.

+ * 2. other client can send request to disable gfx off feature, the request should be honored.

+ * 3. other client can cancel their request of disable gfx off feature

+ * 4. other client should not send request to enable gfx off feature before disable gfx off feature.

+ *

+ * gfx off allow will be delayed by GFX_OFF_DELAY_ENABLE ms.

+ */

+void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)

+{

+ /* If going to s2idle, no need to wait */

+ bool no_delay = adev->in_s0ix ? true : false;

+

+ amdgpu_gfx_do_off_ctrl(adev, enable, no_delay);

+}

+

+/* amdgpu_gfx_off_ctrl_immediate - Handle gfx off feature enable/disable

+ *

+ * @adev: amdgpu_device pointer

+ * @bool enable true: enable gfx off feature, false: disable gfx off feature

+ *

+ * 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled.

+ * 2. other client can send request to disable gfx off feature, the request should be honored.

+ * 3. other client can cancel their request of disable gfx off feature

+ * 4. other client should not send request to enable gfx off feature before disable gfx off feature.

+ *

+ * gfx off allow will be issued immediately.

+ */

+void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable)

+{

+ amdgpu_gfx_do_off_ctrl(adev, enable, true);

+}

+

+ for (i = 0; i < num_partitions; i++)

+ amdgpu_mes_update_enforce_isolation(adev);

+

+ amdgpu_amdkfd_start_sched(adev, idx);

+ adev->gfx.kfd_sch_inactive[idx] = false;

+void amdgpu_gfx_profile_idle_work_handler(struct work_struct *work)

+{

+ struct amdgpu_device *adev =

+ container_of(work, struct amdgpu_device, gfx.idle_work.work);

+ enum PP_SMC_POWER_PROFILE profile;

+ u32 i, fences = 0;

+ int r;

+

+ if (adev->gfx.num_gfx_rings)

+ profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D;

+ else

+ profile = PP_SMC_POWER_PROFILE_COMPUTE;

+

+ for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i)

+ fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]);

+ for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i)

+ fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]);

+ if (!fences && !atomic_read(&adev->gfx.total_submission_cnt)) {

+ mutex_lock(&adev->gfx.workload_profile_mutex);

+ if (adev->gfx.workload_profile_active) {

+ r = amdgpu_dpm_switch_power_profile(adev, profile, false);

+ if (r)

+ dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r,

+ profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ?

+ "fullscreen 3D" : "compute");

+ adev->gfx.workload_profile_active = false;

+ }

+ mutex_unlock(&adev->gfx.workload_profile_mutex);

+ } else {

+ schedule_delayed_work(&adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT);

+ }

+}

+

+void amdgpu_gfx_profile_ring_begin_use(struct amdgpu_ring *ring)

+{

+ struct amdgpu_device *adev = ring->adev;

+ enum PP_SMC_POWER_PROFILE profile;

+ int r;

+

+ if (adev->gfx.num_gfx_rings)

+ profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D;

+ else

+ profile = PP_SMC_POWER_PROFILE_COMPUTE;

+

+ atomic_inc(&adev->gfx.total_submission_cnt);

+

+ cancel_delayed_work_sync(&adev->gfx.idle_work);

+

+ /* We can safely return early here because we've cancelled the

+ * the delayed work so there is no one else to set it to false

+ * and we don't care if someone else sets it to true.

+ */

+ if (adev->gfx.workload_profile_active)

+ return;

+

+ mutex_lock(&adev->gfx.workload_profile_mutex);

+ if (!adev->gfx.workload_profile_active) {

+ r = amdgpu_dpm_switch_power_profile(adev, profile, true);

+ if (r)

+ dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r,

+ profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ?

+ "fullscreen 3D" : "compute");

+ adev->gfx.workload_profile_active = true;

+ }

+ mutex_unlock(&adev->gfx.workload_profile_mutex);

+}

+

+void amdgpu_gfx_profile_ring_end_use(struct amdgpu_ring *ring)

+{

+ atomic_dec(&ring->adev->gfx.total_submission_cnt);

+

+ schedule_delayed_work(&ring->adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT);

+}

+

+/* 1 second timeout */

+#define GFX_PROFILE_IDLE_TIMEOUT msecs_to_jiffies(1000)

+

+

+ atomic_t total_submission_cnt;

+ struct delayed_work idle_work;

+ bool workload_profile_active;

+ struct mutex workload_profile_mutex;

+void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable);

+

+void amdgpu_gfx_profile_idle_work_handler(struct work_struct *work);

+void amdgpu_gfx_profile_ring_begin_use(struct amdgpu_ring *ring);

+void amdgpu_gfx_profile_ring_end_use(struct amdgpu_ring *ring);

+

+ * Function will try to place GART before or after VRAM.

+ struct amdgpu_ring *shared_ring;

+ ring == &adev->umsch_mm.ring ||

+ ring == &adev->cper.ring_buf)

+ continue;

+

+ /* Skip if the ring is a shared ring */

+ if (amdgpu_sdma_is_shared_inv_eng(adev, ring))

+ /* SDMA has a special packet which allows it to use the same

+ * invalidation engine for all the rings in one instance.

+ * Therefore, we do not allocate a separate VM invalidation engine

+ * for SDMA page rings. Instead, they share the VM invalidation

+ * engine with the SDMA gfx ring. This change ensures efficient

+ * resource management and avoids the issue of insufficient VM

+ * invalidation engines.

+ */

+ shared_ring = amdgpu_sdma_get_shared_ring(adev, ring);

+ if (shared_ring) {

+ shared_ring->vm_inv_eng = ring->vm_inv_eng;

+ dev_info(adev->dev, "ring %s shares VM invalidation engine %u with ring %s on hub %u\n",

+ ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub);

+ continue;

+ }

+ case IP_VERSION(11, 5, 3):

+ gc_ver == IP_VERSION(9, 5, 0) ||

+#include "amdgpu_xgmi.h"

+void amdgpu_i2c_init(struct amdgpu_device *adev)

+{

+ if (!adev->is_atom_fw) {

+ if (!amdgpu_device_has_dc_support(adev)) {

+ amdgpu_atombios_i2c_init(adev);

+ } else {

+ switch (adev->asic_type) {

+ case CHIP_POLARIS10:

+ case CHIP_POLARIS11:

+ case CHIP_POLARIS12:

+ amdgpu_atombios_oem_i2c_init(adev, 0x97);

+ break;

+ default:

+ break;

+ }

+ }

+ }

+}

+

+void amdgpu_i2c_init(struct amdgpu_device *adev);

+ fences = kmalloc_array(id_mgr->num_ids, sizeof(void *), GFP_NOWAIT);

+ !dma_fence_is_signaled((*id)->last_flush)))

+ needs_flush = true;

+

+ if ((*id)->owner != vm->immediate.fence_context ||

+ (!adev->vm_manager.concurrent_flush && needs_flush)) {

+ /* Don't use per engine and per process VMID at the

+ * same time

+ if (adev->vm_manager.concurrent_flush)

+ ring = NULL;

+

+ /* to prevent one context starved by another context */

+ (*id)->pd_gpu_addr = 0;

+ tmp = amdgpu_sync_peek_fence(&(*id)->active, ring);

+ if (tmp) {

+ *fence = dma_fence_get(tmp);

+ r = amdgpu_sync_fence(&(*id)->active, &job->base.s_fence->finished,

+ GFP_NOWAIT);

+ &job->base.s_fence->finished,

+ GFP_NOWAIT);

+ if (amdgpu_vmid_uses_reserved(vm, vmhub)) {

+ &job->base.s_fence->finished,

+ GFP_NOWAIT);

+bool amdgpu_vmid_uses_reserved(struct amdgpu_vm *vm, unsigned int vmhub)

+ return vm->reserved_vmid[vmhub];

+bool amdgpu_vmid_uses_reserved(struct amdgpu_vm *vm, unsigned int vmhub);

+#define amdgpu_ih_ts_after_or_equal(t1, t2) \

+ (((int64_t)((t2) << 16) - (int64_t)((t1) << 16)) >= 0LL)

+

+static bool isp_is_idle(struct amdgpu_ip_block *ip_block)

+ struct resource *isp_gpio_res;

+ if (unlikely(adev->debug_disable_gpu_ring_reset)) {

+ dev_err(adev->dev, "Ring reset disabled by debug mask\n");

+ } else if (amdgpu_gpu_recovery && ring->funcs->reset) {

+ bool is_guilty;

+

+

+ /* for engine resets, we need to reset the engine,

+ * but individual queues may be unaffected.

+ * check here to make sure the accounting is correct.

+ */

+ if (ring->funcs->is_guilty)

+ is_guilty = ring->funcs->is_guilty(ring);

+ else

+ is_guilty = true;

+

+ if (is_guilty)

+ dma_fence_set_error(&s_job->s_fence->finished, -ETIME);

+

+ if (is_guilty) {

+ atomic_inc(&ring->adev->gpu_reset_counter);

+ amdgpu_fence_driver_force_completion(ring);

+ }

+ dev_err(adev->dev, "Ring %s reset succeeded\n", ring->sched.name);

+ drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE);

+ dma_fence_set_error(&s_job->s_fence->finished, -ETIME);

+ struct dma_fence *fence;

+ if (job->gang_submit) {

+ if (fence)

+ return fence;

+ }

+ fence = amdgpu_device_enforce_isolation(ring->adev, ring, job);

+ if (fence)

+ return fence;

+

+ if (job->vm && !job->vmid) {

+ return fence;

+ return NULL;

+/*

+ * This is a duplicate function from DRM scheduler sched_internal.h.

+ * Plan is to remove it when amdgpu_job_stop_all_jobs_on_sched is removed, due

+ * latter being incorrect and racy.

+ *

+ * See https://lore.kernel.org/amd-gfx/44edde63-7181-44fb-a4f7-94e50514f539@amd.com/

+ */

+static struct drm_sched_job *

+drm_sched_entity_queue_pop(struct drm_sched_entity *entity)

+{

+ struct spsc_node *node;

+

+ node = spsc_queue_pop(&entity->job_queue);

+ if (!node)

+ return NULL;

+

+ return container_of(node, struct drm_sched_job, queue_node);

+}

+ while ((s_job = drm_sched_entity_queue_pop(s_entity))) {

+static void amdgpu_jpeg_reg_dump_fini(struct amdgpu_device *adev);

+ if (adev->jpeg.reg_list)

+ amdgpu_jpeg_reg_dump_fini(adev);

+

+

+int amdgpu_jpeg_reg_dump_init(struct amdgpu_device *adev,

+ const struct amdgpu_hwip_reg_entry *reg, u32 count)

+{

+ adev->jpeg.ip_dump = kcalloc(adev->jpeg.num_jpeg_inst * count,

+ sizeof(uint32_t), GFP_KERNEL);

+ if (!adev->jpeg.ip_dump) {

+ DRM_ERROR("Failed to allocate memory for JPEG IP Dump\n");

+ return -ENOMEM;

+ }

+ adev->jpeg.reg_list = reg;

+ adev->jpeg.reg_count = count;

+

+ return 0;

+}

+

+static void amdgpu_jpeg_reg_dump_fini(struct amdgpu_device *adev)

+{

+ kfree(adev->jpeg.ip_dump);

+ adev->jpeg.reg_list = NULL;

+ adev->jpeg.reg_count = 0;

+}

+

+void amdgpu_jpeg_dump_ip_state(struct amdgpu_ip_block *ip_block)

+{

+ struct amdgpu_device *adev = ip_block->adev;

+ u32 inst_off, inst_id, is_powered;

+ int i, j;

+

+ if (!adev->jpeg.ip_dump)

+ return;

+

+ for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {

+ if (adev->jpeg.harvest_config & (1 << i))

+ continue;

+

+ inst_id = GET_INST(JPEG, i);

+ inst_off = i * adev->jpeg.reg_count;

+ /* check power status from UVD_JPEG_POWER_STATUS */

+ adev->jpeg.ip_dump[inst_off] =

+ RREG32(SOC15_REG_ENTRY_OFFSET_INST(adev->jpeg.reg_list[0],

+ inst_id));

+ is_powered = ((adev->jpeg.ip_dump[inst_off] & 0x1) != 1);

+

+ if (is_powered)

+ for (j = 1; j < adev->jpeg.reg_count; j++)

+ adev->jpeg.ip_dump[inst_off + j] =

+ RREG32(SOC15_REG_ENTRY_OFFSET_INST(adev->jpeg.reg_list[j],

+ inst_id));

+ }

+}

+

+void amdgpu_jpeg_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)

+{

+ struct amdgpu_device *adev = ip_block->adev;

+ u32 inst_off, is_powered;

+ int i, j;

+

+ if (!adev->jpeg.ip_dump)

+ return;

+

+ drm_printf(p, "num_instances:%d\n", adev->jpeg.num_jpeg_inst);

+ for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {

+ if (adev->jpeg.harvest_config & (1 << i)) {

+ drm_printf(p, "\nHarvested Instance:JPEG%d Skipping dump\n", i);

+ continue;

+ }

+

+ inst_off = i * adev->jpeg.reg_count;

+ is_powered = ((adev->jpeg.ip_dump[inst_off] & 0x1) != 1);

+

+ if (is_powered) {

+ drm_printf(p, "Active Instance:JPEG%d\n", i);

+ for (j = 0; j < adev->jpeg.reg_count; j++)

+ drm_printf(p, "%-50s \t 0x%08x\n", adev->jpeg.reg_list[j].reg_name,

+ adev->jpeg.ip_dump[inst_off + j]);

+ } else

+ drm_printf(p, "\nInactive Instance:JPEG%d\n", i);

+ }

+}

+struct amdgpu_hwip_reg_entry;

+

+enum amdgpu_jpeg_caps {

+ AMDGPU_JPEG_RRMT_ENABLED,

+};

+

+#define AMDGPU_JPEG_CAPS(caps) BIT(AMDGPU_JPEG_##caps)

+

+ uint32_t caps;

+ u32 *ip_dump;

+ u32 reg_count;

+ const struct amdgpu_hwip_reg_entry *reg_list;

+int amdgpu_jpeg_reg_dump_init(struct amdgpu_device *adev,

+ const struct amdgpu_hwip_reg_entry *reg, u32 count);

+void amdgpu_jpeg_dump_ip_state(struct amdgpu_ip_block *ip_block);

+void amdgpu_jpeg_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p);

+ for (j = 0; j < adev->vcn.inst[i].num_enc_rings; j++)

+ if (amdgpu_passthrough(adev))

+ dev_info->ids_flags |= (AMDGPU_IDS_FLAGS_MODE_PT <<

+ AMDGPU_IDS_FLAGS_MODE_SHIFT) &

+ AMDGPU_IDS_FLAGS_MODE_MASK;

+ else if (amdgpu_sriov_vf(adev))

+ dev_info->ids_flags |= (AMDGPU_IDS_FLAGS_MODE_VF <<

+ AMDGPU_IDS_FLAGS_MODE_SHIFT) &

+ AMDGPU_IDS_FLAGS_MODE_MASK;

+

+ if (i >= (adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_mec))

+ break;

+ if (i >= adev->sdma.num_instances)

+ break;

+ adev->mes.sdma_hqd_mask[i] = 0xfc;

+ amdgpu_sync_fence(&sync, bo_va->last_pt_update, GFP_KERNEL);

+ amdgpu_sync_fence(&sync, vm->last_update, GFP_KERNEL);

+static int amdgpu_mes_set_enforce_isolation(struct amdgpu_device *adev,

+ uint32_t node_id, bool enable)

+int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev)

+{

+ int i, r = 0;

+

+ if (adev->enable_mes && adev->gfx.enable_cleaner_shader) {

+ mutex_lock(&adev->enforce_isolation_mutex);

+ for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) {

+ if (adev->enforce_isolation[i])

+ r |= amdgpu_mes_set_enforce_isolation(adev, i, true);

+ else

+ r |= amdgpu_mes_set_enforce_isolation(adev, i, false);

+ }

+ mutex_unlock(&adev->enforce_isolation_mutex);

+ }

+ return r;

+}

+

+enum amdgpu_mes_pipe {

+ struct amdgpu_bo *resource_1[AMDGPU_MAX_MES_PIPES];

+ uint64_t resource_1_gpu_addr[AMDGPU_MAX_MES_PIPES];

+ void *resource_1_addr[AMDGPU_MAX_MES_PIPES];

+int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev);

+ bool oem;

+ /*

+ * We lock the private dma_resv object here and since the BO is about to

+ * be released nobody else should have a pointer to it.

+ * So when this locking here fails something is wrong with the reference

+ * counting.

+ */

+ if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv)))

+ return;

+

+ amdgpu_amdkfd_remove_all_eviction_fences(abo);

+ goto out;

+ r = dma_resv_reserve_fences(&bo->base._resv, 1);

+ if (r)

+ goto out;

+ r = amdgpu_fill_buffer(abo, 0, &bo->base._resv, &fence, true);

+ if (WARN_ON(r))

+ goto out;

+

+ amdgpu_vram_mgr_set_cleared(bo->resource);

+ dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL);

+ dma_fence_put(fence);

+out:

+ dma_resv_unlock(&bo->base._resv);

+#define AMD_VBIOS_FILE_MAX_SIZE_B (1024*1024*16)

+ case IP_VERSION(13, 0, 12):

+ ret = psp_init_ta_microcode(psp, ucode_prefix);

+ break;

+ case IP_VERSION(11, 5, 2):

+ case IP_VERSION(13, 0, 12):

+ psp_v13_0_set_psp_funcs(psp);

+ psp->autoload_supported = false;

+ adev->psp.sup_ifwi_up = !amdgpu_sriov_vf(adev);

+ break;

+ case IP_VERSION(14, 0, 5):

+ psp_v14_0_set_psp_funcs(psp);

+ psp->boot_time_tmr = false;

+ break;

+ kfree(psp->cmd);

+ psp->cmd = NULL;

+ if (boot_cfg == 1 && !adev->ras_default_ecc_enabled &&

+ amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC)) {

+ dev_warn(adev->dev, "GECC is currently enabled, which may affect performance\n");

+ dev_warn(adev->dev,

+ "To disable GECC, please reboot the system and load the amdgpu driver with the parameter amdgpu_ras_enable=0\n");

+ if ((adev->ras_default_ecc_enabled || amdgpu_ras_enable == 1) &&

+ amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC)) {

+ if (boot_cfg == 1) {

+ dev_info(adev->dev, "GECC is enabled\n");

+ } else {

+ /* enable GECC in next boot cycle if it is disabled

+ * in boot config, or force enable GECC if failed to

+ * get boot configuration

+ */

+ ret = psp_boot_config_set(adev, BOOT_CONFIG_GECC);

+ if (ret)

+ dev_warn(adev->dev, "PSP set boot config failed\n");

+ else

+ dev_warn(adev->dev, "GECC will be enabled in next boot cycle\n");

+ }

+ if (!boot_cfg) {

+ if (!adev->ras_default_ecc_enabled &&

+ amdgpu_ras_enable != 1 &&

+ amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC))

+ dev_warn(adev->dev, "GECC is disabled, set amdgpu_ras_enable=1 to enable GECC in next boot cycle if needed\n");

+ else

+ dev_info(adev->dev, "GECC is disabled\n");

+ } else {

+ /* disable GECC in next boot cycle if ras is

+ * disabled by module parameter amdgpu_ras_enable

+ * and/or amdgpu_ras_mask, or boot_config_get call

+ * is failed

+ */

+ ret = psp_boot_config_set(adev, 0);

+ if (ret)

+ dev_warn(adev->dev, "PSP set boot config failed\n");

+ else

+ dev_warn(adev->dev, "GECC will be disabled in next boot cycle if set amdgpu_ras_enable and/or amdgpu_ras_mask to 0x0\n");

+ }

+ ras_cmd->ras_in_message.init_flags.active_umc_mask = adev->umc.active_mask;

+ } else {

+ dev_err(adev->dev, "fail to initialize cap microcode\n");

+ goto out;

+ if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block))

+ return 0;

+

+static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev,

+ struct eeprom_table_record *bps, int count)

+{

+ int j;

+ struct amdgpu_ras *con = amdgpu_ras_get_context(adev);

+ struct ras_err_handler_data *data = con->eh_data;

+

+ for (j = 0; j < count; j++) {

+ if (amdgpu_ras_check_bad_page_unlock(con,

+ bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT))

+ continue;

+

+ if (!data->space_left &&

+ amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {

+ return -ENOMEM;

+ }

+

+ amdgpu_ras_reserve_page(adev, bps[j].retired_page);

+

+ memcpy(&data->bps[data->count], &(bps[j]),

+ sizeof(struct eeprom_table_record));

+ data->count++;

+ data->space_left--;

+ }

+

+ return 0;

+}

+

+static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev,

+ struct eeprom_table_record *bps, struct ras_err_data *err_data,

+ enum amdgpu_memory_partition nps)

+{

+ int i = 0;

+ enum amdgpu_memory_partition save_nps;

+

+ save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;

+

+ /*old asics just have pa in eeprom*/

+ if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {

+ memcpy(err_data->err_addr, bps,

+ sizeof(struct eeprom_table_record) * adev->umc.retire_unit);

+ goto out;

+ }

+

+ for (i = 0; i < adev->umc.retire_unit; i++)

+ bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);

+

+ if (save_nps) {

+ if (save_nps == nps) {

+ if (amdgpu_umc_pages_in_a_row(adev, err_data,

+ bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT))

+ return -EINVAL;

+ } else {

+ if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data))

+ return -EINVAL;

+ }

+ } else {

+ if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) {

+ if (nps == AMDGPU_NPS1_PARTITION_MODE)

+ memcpy(err_data->err_addr, bps,

+ sizeof(struct eeprom_table_record) * adev->umc.retire_unit);

+ else

+ return -EOPNOTSUPP;

+ }

+ }

+

+out:

+ return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit);

+}

+

+static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev,

+ struct eeprom_table_record *bps, struct ras_err_data *err_data,

+ enum amdgpu_memory_partition nps)

+{

+ enum amdgpu_memory_partition save_nps;

+

+ save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;

+ bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);

+

+ if (save_nps == nps) {

+ if (amdgpu_umc_pages_in_a_row(adev, err_data,

+ bps->retired_page << AMDGPU_GPU_PAGE_SHIFT))

+ return -EINVAL;

+ } else {

+ if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data))

+ return -EINVAL;

+ }

+ return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr,

+ adev->umc.retire_unit);

+}

+

+ uint32_t i;

+ return -ENOMEM;

+

+ if (from_rom) {

+ for (i = 0; i < pages; i++) {

+ if (control->ras_num_recs - i >= adev->umc.retire_unit) {

+ if ((bps[i].address == bps[i + 1].address) &&

+ (bps[i].mem_channel == bps[i + 1].mem_channel)) {

+ //deal with retire_unit records a time

+ ret = __amdgpu_ras_convert_rec_array_from_rom(adev,

+ &bps[i], &err_data, nps);

+ if (ret)

+ i += (adev->umc.retire_unit - 1);

+ break;

+ break;

+ for (; i < pages; i++) {

+ ret = __amdgpu_ras_convert_rec_from_rom(adev,

+ &bps[i], &err_data, nps);

+ if (ret)

+ } else {

+ ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages);

+ /*old asics only save pa to eeprom like before*/

+ if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {

+ &data->bps[bad_page_num], save_count)) {

+ &data->bps[bad_page_num +

+ i * adev->umc.retire_unit], 1)) {

+ int ret, i = 0;

+ if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {

+ for (i = 0; i < control->ras_num_recs; i++) {

+ if ((control->ras_num_recs - i) >= adev->umc.retire_unit) {

+ if ((bps[i].address == bps[i + 1].address) &&

+ (bps[i].mem_channel == bps[i + 1].mem_channel)) {

+ control->ras_num_pa_recs += adev->umc.retire_unit;

+ i += (adev->umc.retire_unit - 1);

+ } else {

+ control->ras_num_mca_recs +=

+ (control->ras_num_recs - i);

+ break;

+ }

+ } else {

+ control->ras_num_mca_recs += (control->ras_num_recs - i);

+ break;

+ }

+ }

+ * amdgpu_bad_page_threshold is used to config

+ * the threshold for the number of bad pages.

+ * -1: Threshold is set to default value

+ * Driver will issue a warning message when threshold is reached

+ * and continue runtime services.

+ * 0: Disable bad page retirement

+ * Driver will not retire bad pages

+ * which is intended for debugging purpose.

+ * -2: Threshold is determined by a formula

+ * that assumes 1 bad page per 100M of local memory.

+ * Driver will continue runtime services when threhold is reached.

+ * 0 < threshold < max number of bad page records in EEPROM,

+ * A user-defined threshold is set

+ * Driver will halt runtime services when this custom threshold is reached.

+ if (amdgpu_bad_page_threshold == -2) {

+ } else if (amdgpu_bad_page_threshold == -1) {

+ con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4;

+ control->ras_num_pa_recs = control->ras_num_recs;

+

+ /* The format action is only applied to new ASICs */

+ if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 &&

+ control->tbl_hdr.version < RAS_TABLE_VER_V3)

+ if (!amdgpu_ras_eeprom_reset_table(control))

+ if (amdgpu_ras_save_bad_pages(adev, NULL))

+ dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n");

+ /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */

+ adev->aca.is_enabled =

+ (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||

+ amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||

+ amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14));

+ con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT;

+ break;

+ con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1);

+ if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error))

+ "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n",

+#define AMDGPU_RAS_GPU_ERR_GENERIC(x) AMDGPU_GET_REG_FIELD(x, 30, 30)

+#define AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT (16ULL << 20)

+ case IP_VERSION(13, 0, 12):

+ if (adev->bios && amdgpu_atomfirmware_ras_rom_addr(adev, &i2c_addr)) {

+ case IP_VERSION(13, 0, 12):

+ case IP_VERSION(12, 0, 0):

+ hdr->version = RAS_TABLE_VER_V3;

+ return;

+ if (hdr->version >= RAS_TABLE_VER_V2_1) {

+ if (hdr->version >= RAS_TABLE_VER_V2_1)

+ if (con->eeprom_control.ras_num_bad_pages > con->bad_page_cnt_threshold)

+ con->eeprom_control.ras_num_bad_pages, con->bad_page_cnt_threshold);

+ if ((amdgpu_bad_page_threshold == -1) ||

+ (amdgpu_bad_page_threshold == -2)) {

+ "Please consult AMD Service Action Guide (SAG) for appropriate service procedures.\n");

+ dev_warn(adev->dev,

+ "Please consider adjusting the customized threshold.\n");

+ /*old asics only save pa to eeprom like before*/

+ if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12)

+ control->ras_num_pa_recs += num;

+ control->ras_num_mca_recs += num;

+

+ control->ras_num_bad_pages = control->ras_num_pa_recs +

+ control->ras_num_mca_recs * adev->umc.retire_unit;

+ control->ras_num_bad_pages > ras->bad_page_cnt_threshold) {

+ if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1) {

+ if ((amdgpu_bad_page_threshold != -1) &&

+ (amdgpu_bad_page_threshold != -2))

+ if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1)

+ control->tbl_hdr.version >= RAS_TABLE_VER_V2_1 &&

+ control->ras_num_bad_pages <= ras->bad_page_cnt_threshold)

+ if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1)

+ uint64_t nps = AMDGPU_NPS1_PARTITION_MODE;

+ if (adev->gmc.gmc_funcs->query_mem_partition_mode)

+ nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);

+

+ record[i].retired_page |= (nps << UMC_NPS_SHIFT);

+ record[i].retired_page &= ~(nps << UMC_NPS_SHIFT);

+ if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1)

+ if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1)

+ if (hdr->version >= RAS_TABLE_VER_V2_1) {

+ control->ras_num_mca_recs = 0;

+ control->ras_num_pa_recs = 0;

+ control->ras_num_bad_pages = control->ras_num_pa_recs +

+ control->ras_num_mca_recs * adev->umc.retire_unit;