// SPDX-License-Identifier: MIT /* * Copyright 2015 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: AMD * */ #include "dm_services_types.h" #include "dc.h" #include "dc/dc_dmub_srv.h" #include "dc/dc_stat.h" #include "amdgpu.h" #include "amdgpu_display.h" #include "amdgpu_dm.h" #include "amdgpu_dm_irq.h" #include "dm_helpers.h" #include "amdgpu_dm_crtc.h" #include "amdgpu_dm_hdcp.h" #include "amdgpu_dm_mst_types.h" #include "amdgpu_dm_dmub.h" #include "amdgpu_dm_trace.h" #include "link/protocols/link_dpcd.h" #include "link_service_types.h" #include "ivsrcid/ivsrcid_vislands30.h" #include "ivsrcid/dcn/irqsrcs_dcn_1_0.h" #include "modules/inc/mod_freesync.h" #include /** * DOC: overview * * DM provides another layer of IRQ management on top of what the base driver * already provides. This is something that could be cleaned up, and is a * future TODO item. * * The base driver provides IRQ source registration with DRM, handler * registration into the base driver's IRQ table, and a handler callback * amdgpu_irq_handler(), with which DRM calls on interrupts. This generic * handler looks up the IRQ table, and calls the respective * &amdgpu_irq_src_funcs.process hookups. * * What DM provides on top are two IRQ tables specifically for top-half and * bottom-half IRQ handling, with the bottom-half implementing workqueues: * * - &amdgpu_display_manager.irq_handler_list_high_tab * - &amdgpu_display_manager.irq_handler_list_low_tab * * They override the base driver's IRQ table, and the effect can be seen * in the hooks that DM provides for &amdgpu_irq_src_funcs.process. They * are all set to the DM generic handler amdgpu_dm_irq_handler(), which looks up * DM's IRQ tables. However, in order for base driver to recognize this hook, DM * still needs to register the IRQ with the base driver. See * amdgpu_dm_dce110_register_irq_handlers() and * amdgpu_dm_dcn10_register_irq_handlers(). * * To expose DC's hardware interrupt toggle to the base driver, DM implements * &amdgpu_irq_src_funcs.set hooks. Base driver calls it through * amdgpu_irq_update() to enable or disable the interrupt. */ /****************************************************************************** * Private declarations. *****************************************************************************/ /** * struct amdgpu_dm_irq_handler_data - Data for DM interrupt handlers. * * @list: Linked list entry referencing the next/previous handler * @handler: Handler function * @handler_arg: Argument passed to the handler when triggered * @dm: DM which this handler belongs to * @irq_source: DC interrupt source that this handler is registered for * @work: work struct */ struct amdgpu_dm_irq_handler_data { struct list_head list; interrupt_handler handler; void *handler_arg; struct amdgpu_display_manager *dm; /* DAL irq source which registered for this interrupt. */ enum dc_irq_source irq_source; struct work_struct work; }; #define DM_IRQ_TABLE_LOCK(adev, flags) \ spin_lock_irqsave(&adev->dm.irq_handler_list_table_lock, flags) #define DM_IRQ_TABLE_UNLOCK(adev, flags) \ spin_unlock_irqrestore(&adev->dm.irq_handler_list_table_lock, flags) /****************************************************************************** * Private functions. *****************************************************************************/ static void init_handler_common_data(struct amdgpu_dm_irq_handler_data *hcd, void (*ih)(void *), void *args, struct amdgpu_display_manager *dm) { hcd->handler = ih; hcd->handler_arg = args; hcd->dm = dm; } /** * dm_irq_work_func() - Handle an IRQ outside of the interrupt handler proper. * * @work: work struct */ static void dm_irq_work_func(struct work_struct *work) { struct amdgpu_dm_irq_handler_data *handler_data = container_of(work, struct amdgpu_dm_irq_handler_data, work); handler_data->handler(handler_data->handler_arg); /* Call a DAL subcomponent which registered for interrupt notification * at INTERRUPT_LOW_IRQ_CONTEXT. * (The most common use is HPD interrupt) */ } /* * Remove a handler and return a pointer to handler list from which the * handler was removed. */ static struct list_head *remove_irq_handler(struct amdgpu_device *adev, void *ih, const struct dc_interrupt_params *int_params) { struct list_head *hnd_list; struct list_head *entry, *tmp; struct amdgpu_dm_irq_handler_data *handler; unsigned long irq_table_flags; bool handler_removed = false; enum dc_irq_source irq_source; DM_IRQ_TABLE_LOCK(adev, irq_table_flags); irq_source = int_params->irq_source; switch (int_params->int_context) { case INTERRUPT_HIGH_IRQ_CONTEXT: hnd_list = &adev->dm.irq_handler_list_high_tab[irq_source]; break; case INTERRUPT_LOW_IRQ_CONTEXT: default: hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source]; break; } list_for_each_safe(entry, tmp, hnd_list) { handler = list_entry(entry, struct amdgpu_dm_irq_handler_data, list); if (handler == NULL) continue; if (ih == handler->handler) { /* Found our handler. Remove it from the list. */ list_del(&handler->list); handler_removed = true; break; } } DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); if (!handler_removed) { /* Not necessarily an error - caller may not * know the context. */ return NULL; } if (int_params->int_context == INTERRUPT_LOW_IRQ_CONTEXT) cancel_work_sync(&handler->work); kfree(handler); DRM_DEBUG_KMS( "DM_IRQ: removed irq handler: %p for: dal_src=%d, irq context=%d\n", ih, int_params->irq_source, int_params->int_context); return hnd_list; } static bool validate_irq_registration_params(struct dc_interrupt_params *int_params, void (*ih)(void *)) { if (NULL == int_params || NULL == ih) { DRM_ERROR("DM_IRQ: invalid input!\n"); return false; } if (int_params->int_context >= INTERRUPT_CONTEXT_NUMBER) { DRM_ERROR("DM_IRQ: invalid context: %d!\n", int_params->int_context); return false; } if (!DAL_VALID_IRQ_SRC_NUM(int_params->irq_source)) { DRM_ERROR("DM_IRQ: invalid irq_source: %d!\n", int_params->irq_source); return false; } return true; } static bool validate_irq_unregistration_params(enum dc_irq_source irq_source, irq_handler_idx handler_idx) { if (handler_idx == DAL_INVALID_IRQ_HANDLER_IDX) { DRM_ERROR("DM_IRQ: invalid handler_idx==NULL!\n"); return false; } if (!DAL_VALID_IRQ_SRC_NUM(irq_source)) { DRM_ERROR("DM_IRQ: invalid irq_source:%d!\n", irq_source); return false; } return true; } /****************************************************************************** * Public functions. * * Note: caller is responsible for input validation. *****************************************************************************/ /** * amdgpu_dm_irq_register_interrupt() - Register a handler within DM. * @adev: The base driver device containing the DM device. * @int_params: Interrupt parameters containing the source, and handler context * @ih: Function pointer to the interrupt handler to register * @handler_args: Arguments passed to the handler when the interrupt occurs * * Register an interrupt handler for the given IRQ source, under the given * context. The context can either be high or low. High context handlers are * executed directly within ISR context, while low context is executed within a * workqueue, thereby allowing operations that sleep. * * Registered handlers are called in a FIFO manner, i.e. the most recently * registered handler will be called first. * * Return: Handler data &struct amdgpu_dm_irq_handler_data containing the IRQ * source, handler function, and args */ void *amdgpu_dm_irq_register_interrupt(struct amdgpu_device *adev, struct dc_interrupt_params *int_params, void (*ih)(void *), void *handler_args) { struct list_head *hnd_list; struct amdgpu_dm_irq_handler_data *handler_data; unsigned long irq_table_flags; enum dc_irq_source irq_source; if (!validate_irq_registration_params(int_params, ih)) return DAL_INVALID_IRQ_HANDLER_IDX; handler_data = kzalloc_obj(*handler_data); if (!handler_data) { DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n"); return DAL_INVALID_IRQ_HANDLER_IDX; } init_handler_common_data(handler_data, ih, handler_args, &adev->dm); irq_source = int_params->irq_source; handler_data->irq_source = irq_source; /* Lock the list, add the handler. */ DM_IRQ_TABLE_LOCK(adev, irq_table_flags); switch (int_params->int_context) { case INTERRUPT_HIGH_IRQ_CONTEXT: hnd_list = &adev->dm.irq_handler_list_high_tab[irq_source]; break; case INTERRUPT_LOW_IRQ_CONTEXT: default: hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source]; INIT_WORK(&handler_data->work, dm_irq_work_func); break; } list_add_tail(&handler_data->list, hnd_list); DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); /* This pointer will be stored by code which requested interrupt * registration. * The same pointer will be needed in order to unregister the * interrupt. */ DRM_DEBUG_KMS( "DM_IRQ: added irq handler: %p for: dal_src=%d, irq context=%d\n", handler_data, irq_source, int_params->int_context); return handler_data; } EXPORT_IF_KUNIT(amdgpu_dm_irq_register_interrupt); /** * amdgpu_dm_irq_unregister_interrupt() - Remove a handler from the DM IRQ table * @adev: The base driver device containing the DM device * @irq_source: IRQ source to remove the given handler from * @ih: Function pointer to the interrupt handler to unregister * * Go through both low and high context IRQ tables, and find the given handler * for the given irq source. If found, remove it. Otherwise, do nothing. */ void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev, enum dc_irq_source irq_source, void *ih) { struct list_head *handler_list; struct dc_interrupt_params int_params; int i; if (!validate_irq_unregistration_params(irq_source, ih)) return; memset(&int_params, 0, sizeof(int_params)); int_params.irq_source = irq_source; for (i = 0; i < INTERRUPT_CONTEXT_NUMBER; i++) { int_params.int_context = i; handler_list = remove_irq_handler(adev, ih, &int_params); if (handler_list != NULL) break; } if (handler_list == NULL) { /* If we got here, it means we searched all irq contexts * for this irq source, but the handler was not found. */ DRM_ERROR( "DM_IRQ: failed to find irq handler:%p for irq_source:%d!\n", ih, irq_source); } } EXPORT_IF_KUNIT(amdgpu_dm_irq_unregister_interrupt); /** * amdgpu_dm_irq_init() - Initialize DM IRQ management * @adev: The base driver device containing the DM device * * Initialize DM's high and low context IRQ tables. * * The N by M table contains N IRQ sources, with M * &struct amdgpu_dm_irq_handler_data hooked together in a linked list. The * list_heads are initialized here. When an interrupt n is triggered, all m * handlers are called in sequence, FIFO according to registration order. * * The low context table requires special steps to initialize, since handlers * will be deferred to a workqueue. See &struct irq_list_head. */ int amdgpu_dm_irq_init(struct amdgpu_device *adev) { int src; struct list_head *lh; DRM_DEBUG_KMS("DM_IRQ\n"); spin_lock_init(&adev->dm.irq_handler_list_table_lock); spin_lock_init(&adev->dm.irq_reg_lock); adev->dm.irq_wq = alloc_workqueue("amdgpu_dm_irq", WQ_UNBOUND | WQ_HIGHPRI, 0); if (!adev->dm.irq_wq) return -ENOMEM; adev->dm.vmin_vmax_wq = alloc_workqueue("amdgpu_dm_vmin_vmax", WQ_UNBOUND, 0); if (!adev->dm.vmin_vmax_wq) { destroy_workqueue(adev->dm.irq_wq); adev->dm.irq_wq = NULL; return -ENOMEM; } for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) { /* low context handler list init */ lh = &adev->dm.irq_handler_list_low_tab[src]; INIT_LIST_HEAD(lh); /* high context handler init */ INIT_LIST_HEAD(&adev->dm.irq_handler_list_high_tab[src]); } return 0; } EXPORT_IF_KUNIT(amdgpu_dm_irq_init); /** * amdgpu_dm_irq_fini() - Tear down DM IRQ management * @adev: The base driver device containing the DM device * * Removes all handlers from the IRQ tables under the spinlock, cancels * pending work items, and deallocates all handler data. */ void amdgpu_dm_irq_fini(struct amdgpu_device *adev) { int src; LIST_HEAD(low_handlers); LIST_HEAD(high_handlers); struct list_head *entry, *tmp; struct amdgpu_dm_irq_handler_data *handler; unsigned long irq_table_flags; DRM_DEBUG_KMS("DM_IRQ: releasing resources.\n"); for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) { DM_IRQ_TABLE_LOCK(adev, irq_table_flags); /* * Move all handlers from the low and high context tables to * temporary lists under the lock. This prevents the ISR from * finding them while we process them outside the lock. */ list_splice_init(&adev->dm.irq_handler_list_low_tab[src], &low_handlers); list_splice_init(&adev->dm.irq_handler_list_high_tab[src], &high_handlers); DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); /* * Cancel all pending work for the low-context handlers * outside the lock. cancel_work_sync() may sleep and waits * until any running work completes, preventing UAF. */ list_for_each_safe(entry, tmp, &low_handlers) { handler = list_entry(entry, struct amdgpu_dm_irq_handler_data, list); cancel_work_sync(&handler->work); } /* * High-context handlers are executed synchronously within ISR * context (see amdgpu_dm_irq_immediate_work()) and have no * work_struct, so there is no pending work to cancel here. * They will be freed along with low_handlers after the loop. */ } /* Deallocate all handlers. */ list_for_each_safe(entry, tmp, &low_handlers) { handler = list_entry(entry, struct amdgpu_dm_irq_handler_data, list); list_del(&handler->list); kfree(handler); } list_for_each_safe(entry, tmp, &high_handlers) { handler = list_entry(entry, struct amdgpu_dm_irq_handler_data, list); list_del(&handler->list); kfree(handler); } if (adev->dm.vmin_vmax_wq) { destroy_workqueue(adev->dm.vmin_vmax_wq); adev->dm.vmin_vmax_wq = NULL; } if (adev->dm.irq_wq) { destroy_workqueue(adev->dm.irq_wq); adev->dm.irq_wq = NULL; } } EXPORT_IF_KUNIT(amdgpu_dm_irq_fini); void amdgpu_dm_irq_suspend(struct amdgpu_device *adev) { struct drm_device *dev = adev_to_drm(adev); int src; struct list_head *hnd_list_l; unsigned long irq_table_flags; struct list_head *entry, *tmp; struct amdgpu_dm_irq_handler_data *handler; DM_IRQ_TABLE_LOCK(adev, irq_table_flags); DRM_DEBUG_KMS("DM_IRQ: suspend\n"); /** * Disable HW interrupt for HPD and HPDRX only since FLIP and VBLANK * will be disabled from manage_dm_interrupts on disable CRTC. * * Disable the HW interrupt first, then flush any pending work. Since * the HW interrupt is disabled under the lock, no new IRQ can be * generated after the disable completes. Any work already queued by an * in-flight ISR will be flushed below. */ for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6RX; src++) { hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; amdgpu_dm_irq_set(adev, src, false); DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); if (!list_empty(hnd_list_l)) { list_for_each_safe(entry, tmp, hnd_list_l) { handler = list_entry( entry, struct amdgpu_dm_irq_handler_data, list); flush_work(&handler->work); } } DM_IRQ_TABLE_LOCK(adev, irq_table_flags); } DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); if (dev->mode_config.poll_enabled) drm_kms_helper_poll_disable(dev); } EXPORT_IF_KUNIT(amdgpu_dm_irq_suspend); void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev) { int src; struct list_head *hnd_list_h, *hnd_list_l; unsigned long irq_table_flags; DM_IRQ_TABLE_LOCK(adev, irq_table_flags); drm_dbg(adev_to_drm(adev), "DM_IRQ: early resume\n"); /* re-enable short pulse interrupts HW interrupt */ for (src = DC_IRQ_SOURCE_HPD1RX; src <= DC_IRQ_SOURCE_HPD6RX; src++) { hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) amdgpu_dm_irq_set(adev, src, true); } DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); } EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_early); void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev) { struct drm_device *dev = adev_to_drm(adev); int src; struct list_head *hnd_list_h, *hnd_list_l; unsigned long irq_table_flags; DM_IRQ_TABLE_LOCK(adev, irq_table_flags); drm_dbg(adev_to_drm(adev), "DM_IRQ: resume\n"); /** * Renable HW interrupt for HPD and only since FLIP and VBLANK * will be enabled from manage_dm_interrupts on enable CRTC. */ for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6; src++) { hnd_list_l = &adev->dm.irq_handler_list_low_tab[src]; hnd_list_h = &adev->dm.irq_handler_list_high_tab[src]; if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h)) amdgpu_dm_irq_set(adev, src, true); } DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); if (dev->mode_config.poll_enabled) drm_kms_helper_poll_enable(dev); } EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_late); /* * amdgpu_dm_irq_schedule_work - schedule all work items registered for the * "irq_source". */ STATIC_IFN_KUNIT void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev, enum dc_irq_source irq_source) { struct list_head *handler_list = &adev->dm.irq_handler_list_low_tab[irq_source]; struct amdgpu_dm_irq_handler_data *handler_data; bool work_queued = false; unsigned long irq_table_flags; DM_IRQ_TABLE_LOCK(adev, irq_table_flags); if (!adev->dm.irq_wq) goto out_unlock; if (list_empty(handler_list)) goto out_unlock; list_for_each_entry(handler_data, handler_list, list) { if (queue_work(adev->dm.irq_wq, &handler_data->work)) { work_queued = true; break; } } if (!work_queued) { struct amdgpu_dm_irq_handler_data *handler_data_add; /*get the amdgpu_dm_irq_handler_data of first item pointed by handler_list*/ handler_data = container_of(handler_list->next, struct amdgpu_dm_irq_handler_data, list); /*allocate a new amdgpu_dm_irq_handler_data*/ handler_data_add = kzalloc_obj(*handler_data, GFP_ATOMIC); if (!handler_data_add) { DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n"); goto out_unlock; } /*copy new amdgpu_dm_irq_handler_data members from handler_data*/ handler_data_add->handler = handler_data->handler; handler_data_add->handler_arg = handler_data->handler_arg; handler_data_add->dm = handler_data->dm; handler_data_add->irq_source = irq_source; list_add_tail(&handler_data_add->list, handler_list); INIT_WORK(&handler_data_add->work, dm_irq_work_func); if (queue_work(adev->dm.irq_wq, &handler_data_add->work)) DRM_DEBUG("Queued work for handling interrupt from " "display for IRQ source %d\n", irq_source); else DRM_ERROR("Failed to queue work for handling interrupt " "from display for IRQ source %d\n", irq_source); } out_unlock: DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); } EXPORT_IF_KUNIT(amdgpu_dm_irq_schedule_work); /* * amdgpu_dm_irq_immediate_work * Callback high irq work immediately, don't send to work queue */ STATIC_IFN_KUNIT void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev, enum dc_irq_source irq_source) { struct amdgpu_dm_irq_handler_data *handler_data; unsigned long irq_table_flags; DM_IRQ_TABLE_LOCK(adev, irq_table_flags); list_for_each_entry(handler_data, &adev->dm.irq_handler_list_high_tab[irq_source], list) { /* Call a subcomponent which registered for immediate * interrupt notification */ handler_data->handler(handler_data->handler_arg); } DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags); } EXPORT_IF_KUNIT(amdgpu_dm_irq_immediate_work); bool amdgpu_dm_irq_set(struct amdgpu_device *adev, enum dc_irq_source src, bool enable) { guard(spinlock_irqsave)(&adev->dm.irq_reg_lock); return dc_interrupt_set(adev->dm.dc, src, enable); } EXPORT_IF_KUNIT(amdgpu_dm_irq_set); void amdgpu_dm_irq_ack(struct amdgpu_device *adev, enum dc_irq_source src) { guard(spinlock_irqsave)(&adev->dm.irq_reg_lock); dc_interrupt_ack(adev->dm.dc, src); } EXPORT_IF_KUNIT(amdgpu_dm_irq_ack); /** * amdgpu_dm_irq_handler - Generic DM IRQ handler * @adev: amdgpu base driver device containing the DM device * @source: Unused * @entry: Data about the triggered interrupt * * Calls all registered high irq work immediately, and schedules work for low * irq. The DM IRQ table is used to find the corresponding handlers. */ STATIC_IFN_KUNIT int amdgpu_dm_irq_handler(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { enum dc_irq_source src = dc_interrupt_to_irq_source( adev->dm.dc, entry->src_id, entry->src_data[0]); if (!DAL_VALID_IRQ_SRC_NUM(src)) { DRM_ERROR("DM_IRQ: invalid irq_source:%d!\n", src); return 0; } amdgpu_dm_irq_ack(adev, src); /* Call high irq work immediately */ amdgpu_dm_irq_immediate_work(adev, src); /*Schedule low_irq work */ amdgpu_dm_irq_schedule_work(adev, src); return 0; } EXPORT_IF_KUNIT(amdgpu_dm_irq_handler); STATIC_IFN_KUNIT enum dc_irq_source amdgpu_dm_hpd_to_dal_irq_source(unsigned int type) { switch (type) { case AMDGPU_HPD_1: return DC_IRQ_SOURCE_HPD1; case AMDGPU_HPD_2: return DC_IRQ_SOURCE_HPD2; case AMDGPU_HPD_3: return DC_IRQ_SOURCE_HPD3; case AMDGPU_HPD_4: return DC_IRQ_SOURCE_HPD4; case AMDGPU_HPD_5: return DC_IRQ_SOURCE_HPD5; case AMDGPU_HPD_6: return DC_IRQ_SOURCE_HPD6; default: return DC_IRQ_SOURCE_INVALID; } } EXPORT_IF_KUNIT(amdgpu_dm_hpd_to_dal_irq_source); STATIC_IFN_KUNIT int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int type, enum amdgpu_interrupt_state state) { enum dc_irq_source src = amdgpu_dm_hpd_to_dal_irq_source(type); bool st = (state == AMDGPU_IRQ_STATE_ENABLE); amdgpu_dm_irq_set(adev, src, st); return 0; } EXPORT_IF_KUNIT(amdgpu_dm_set_hpd_irq_state); static inline int dm_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int crtc_id, enum amdgpu_interrupt_state state, const enum irq_type dal_irq_type, const char *func) { bool st; enum dc_irq_source irq_source; struct dc *dc = adev->dm.dc; struct amdgpu_crtc *acrtc = adev->mode_info.crtcs[crtc_id]; if (!acrtc) { DRM_ERROR( "%s: crtc is NULL at id :%d\n", func, crtc_id); return 0; } if (acrtc->otg_inst == -1) return 0; irq_source = dal_irq_type + acrtc->otg_inst; st = (state == AMDGPU_IRQ_STATE_ENABLE); if (dc && dc->caps.ips_support && dc->idle_optimizations_allowed) dc_allow_idle_optimizations(dc, false); amdgpu_dm_irq_set(adev, irq_source, st); return 0; } STATIC_IFN_KUNIT int amdgpu_dm_set_pflip_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int crtc_id, enum amdgpu_interrupt_state state) { return dm_irq_state( adev, source, crtc_id, state, IRQ_TYPE_PFLIP, __func__); } EXPORT_IF_KUNIT(amdgpu_dm_set_pflip_irq_state); STATIC_IFN_KUNIT int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int crtc_id, enum amdgpu_interrupt_state state) { return dm_irq_state( adev, source, crtc_id, state, IRQ_TYPE_VBLANK, __func__); } EXPORT_IF_KUNIT(amdgpu_dm_set_crtc_irq_state); STATIC_IFN_KUNIT int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int crtc_id, enum amdgpu_interrupt_state state) { return dm_irq_state( adev, source, crtc_id, state, IRQ_TYPE_VLINE0, __func__); } EXPORT_IF_KUNIT(amdgpu_dm_set_vline0_irq_state); STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int crtc_id, enum amdgpu_interrupt_state state) { enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX; bool st = (state == AMDGPU_IRQ_STATE_ENABLE); amdgpu_dm_irq_set(adev, irq_source, st); return 0; } EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_outbox_irq_state); STATIC_IFN_KUNIT int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int crtc_id, enum amdgpu_interrupt_state state) { return dm_irq_state( adev, source, crtc_id, state, IRQ_TYPE_VUPDATE, __func__); } EXPORT_IF_KUNIT(amdgpu_dm_set_vupdate_irq_state); STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int type, enum amdgpu_interrupt_state state) { enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX0; bool st = (state == AMDGPU_IRQ_STATE_ENABLE); amdgpu_dm_irq_set(adev, irq_source, st); return 0; } EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_trace_irq_state); static const struct amdgpu_irq_src_funcs dm_crtc_irq_funcs = { .set = amdgpu_dm_set_crtc_irq_state, .process = amdgpu_dm_irq_handler, }; static const struct amdgpu_irq_src_funcs dm_vline0_irq_funcs = { .set = amdgpu_dm_set_vline0_irq_state, .process = amdgpu_dm_irq_handler, }; static const struct amdgpu_irq_src_funcs dm_dmub_outbox_irq_funcs = { .set = amdgpu_dm_set_dmub_outbox_irq_state, .process = amdgpu_dm_irq_handler, }; static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = { .set = amdgpu_dm_set_vupdate_irq_state, .process = amdgpu_dm_irq_handler, }; static const struct amdgpu_irq_src_funcs dm_dmub_trace_irq_funcs = { .set = amdgpu_dm_set_dmub_trace_irq_state, .process = amdgpu_dm_irq_handler, }; static const struct amdgpu_irq_src_funcs dm_pageflip_irq_funcs = { .set = amdgpu_dm_set_pflip_irq_state, .process = amdgpu_dm_irq_handler, }; static const struct amdgpu_irq_src_funcs dm_hpd_irq_funcs = { .set = amdgpu_dm_set_hpd_irq_state, .process = amdgpu_dm_irq_handler, }; void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev) { adev->crtc_irq.num_types = adev->mode_info.num_crtc; adev->crtc_irq.funcs = &dm_crtc_irq_funcs; adev->vline0_irq.num_types = adev->mode_info.num_crtc; adev->vline0_irq.funcs = &dm_vline0_irq_funcs; adev->dmub_outbox_irq.num_types = 1; adev->dmub_outbox_irq.funcs = &dm_dmub_outbox_irq_funcs; adev->vupdate_irq.num_types = adev->mode_info.num_crtc; adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs; adev->dmub_trace_irq.num_types = 1; adev->dmub_trace_irq.funcs = &dm_dmub_trace_irq_funcs; adev->pageflip_irq.num_types = adev->mode_info.num_crtc; adev->pageflip_irq.funcs = &dm_pageflip_irq_funcs; adev->hpd_irq.num_types = adev->mode_info.num_hpd; adev->hpd_irq.funcs = &dm_hpd_irq_funcs; } EXPORT_IF_KUNIT(amdgpu_dm_set_irq_funcs); void amdgpu_dm_outbox_init(struct amdgpu_device *adev) { amdgpu_dm_irq_set(adev, DC_IRQ_SOURCE_DMCUB_OUTBOX, true); } EXPORT_IF_KUNIT(amdgpu_dm_outbox_init); /** * amdgpu_dm_hpd_init - hpd setup callback. * * @adev: amdgpu_device pointer * * Setup the hpd pins used by the card (evergreen+). * Enable the pin, set the polarity, and enable the hpd interrupts. */ void amdgpu_dm_hpd_init(struct amdgpu_device *adev) { struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; int irq_type; int i; bool use_polling = false; /* * First, clear all hpd and hpdrx interrupts, over the links this ASIC * has rather than over the whole enum: the unused slots of the DC * interrupt table hold dummy handlers that assert when touched. */ for (i = 0; i < adev->dm.dc->link_count; i++) { const struct dc_link *link = adev->dm.dc->links[i]; if (link->irq_source_hpd != DC_IRQ_SOURCE_INVALID && !amdgpu_dm_irq_set(adev, link->irq_source_hpd, false)) drm_err(dev, "Failed to clear hpd source=%d on init\n", link->irq_source_hpd); if (link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID && !amdgpu_dm_irq_set(adev, link->irq_source_hpd_rx, false)) drm_err(dev, "Failed to clear hpdrx source=%d on init\n", link->irq_source_hpd_rx); } drm_connector_list_iter_begin(dev, &iter); drm_for_each_connector_iter(connector, &iter) { struct amdgpu_dm_connector *amdgpu_dm_connector; const struct dc_link *dc_link; if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) continue; amdgpu_dm_connector = to_amdgpu_dm_connector(connector); dc_link = amdgpu_dm_connector->dc_link; if (!dc_link) continue; /* * Analog connectors may be hot-plugged unlike other connector * types that don't support HPD. Only poll analog connectors. */ use_polling |= dc_connector_supports_analog(dc_link->link_id.id); /* * Get a base driver irq reference for hpd ints for the lifetime * of dm. Note that only hpd interrupt types are registered with * base driver; hpd_rx types aren't. IOW, amdgpu_irq_get/put on * hpd_rx isn't available. DM currently controls hpd_rx * explicitly with amdgpu_dm_irq_set() */ if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1; /* * TODO: There's a mismatch between mode_info.num_hpd * and what bios reports as the # of connectors with hpd * sources. Since the # of hpd source types registered * with base driver == mode_info.num_hpd, we have to * fallback to amdgpu_dm_irq_set for the remaining types. */ if (irq_type < adev->mode_info.num_hpd) { if (amdgpu_irq_get(adev, &adev->hpd_irq, irq_type)) drm_err(dev, "DM_IRQ: Failed get HPD for source=%d)!\n", dc_link->irq_source_hpd); } else { amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd, true); } } if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx, true); } } drm_connector_list_iter_end(&iter); if (use_polling) drm_kms_helper_poll_init(dev); } EXPORT_IF_KUNIT(amdgpu_dm_hpd_init); /** * amdgpu_dm_hpd_fini - hpd tear down callback. * * @adev: amdgpu_device pointer * * Tear down the hpd pins used by the card (evergreen+). * Disable the hpd interrupts. */ void amdgpu_dm_hpd_fini(struct amdgpu_device *adev) { struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct drm_connector_list_iter iter; int irq_type; drm_connector_list_iter_begin(dev, &iter); drm_for_each_connector_iter(connector, &iter) { struct amdgpu_dm_connector *amdgpu_dm_connector; const struct dc_link *dc_link; if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) continue; amdgpu_dm_connector = to_amdgpu_dm_connector(connector); dc_link = amdgpu_dm_connector->dc_link; if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1; /* TODO: See same TODO in amdgpu_dm_hpd_init() */ if (irq_type < adev->mode_info.num_hpd) { if (amdgpu_irq_put(adev, &adev->hpd_irq, irq_type)) drm_err(dev, "DM_IRQ: Failed put HPD for source=%d!\n", dc_link->irq_source_hpd); } else { amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd, false); } } if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx, false); } } drm_connector_list_iter_end(&iter); if (dev->mode_config.poll_enabled) drm_kms_helper_poll_fini(dev); } EXPORT_IF_KUNIT(amdgpu_dm_hpd_fini); /* ========== HPD handling ========== */ static void force_connector_state( struct amdgpu_dm_connector *aconnector, enum drm_connector_force force_state) { struct drm_connector *connector = &aconnector->base; mutex_lock(&connector->dev->mode_config.mutex); aconnector->base.force = force_state; mutex_unlock(&connector->dev->mode_config.mutex); mutex_lock(&aconnector->hpd_lock); drm_kms_helper_connector_hotplug_event(connector); mutex_unlock(&aconnector->hpd_lock); } STATIC_IFN_KUNIT void dm_handle_hpd_rx_offload_work(struct work_struct *work) { struct hpd_rx_irq_offload_work *offload_work; struct amdgpu_dm_connector *aconnector; struct dc_link *dc_link; struct amdgpu_device *adev; enum dc_connection_type new_connection_type = dc_connection_none; unsigned long flags; union test_response test_response; memset(&test_response, 0, sizeof(test_response)); offload_work = container_of(work, struct hpd_rx_irq_offload_work, work); aconnector = offload_work->offload_wq->aconnector; adev = offload_work->adev; if (!aconnector) { drm_err(adev_to_drm(adev), "Can't retrieve aconnector in hpd_rx_irq_offload_work"); goto skip; } dc_link = aconnector->dc_link; mutex_lock(&aconnector->hpd_lock); if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n"); mutex_unlock(&aconnector->hpd_lock); if (new_connection_type == dc_connection_none) goto skip; if (amdgpu_in_reset(adev)) goto skip; if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT); spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); offload_work->offload_wq->is_handling_mst_msg_rdy_event = false; spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); goto skip; } mutex_lock(&adev->dm.dc_lock); if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) { dc_link_dp_handle_automated_test(dc_link); if (aconnector->timing_changed) { /* force connector disconnect and reconnect */ force_connector_state(aconnector, DRM_FORCE_OFF); msleep(100); force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED); } test_response.bits.ACK = 1; core_link_write_dpcd( dc_link, DP_TEST_RESPONSE, &test_response.raw, sizeof(test_response)); } else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) && dc_link_check_link_loss_status(dc_link, &offload_work->data) && dc_link_dp_allow_hpd_rx_irq(dc_link)) { /* offload_work->data is from handle_hpd_rx_irq-> * schedule_hpd_rx_offload_work.this is defer handle * for hpd short pulse. upon here, link status may be * changed, need get latest link status from dpcd * registers. if link status is good, skip run link * training again. */ union hpd_irq_data irq_data; memset(&irq_data, 0, sizeof(irq_data)); /* before dc_link_dp_handle_link_loss, allow new link lost handle * request be added to work queue if link lost at end of dc_link_ * dp_handle_link_loss */ spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); offload_work->offload_wq->is_handling_link_loss = false; spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) && dc_link_check_link_loss_status(dc_link, &irq_data)) dc_link_dp_handle_link_loss(dc_link); } mutex_unlock(&adev->dm.dc_lock); skip: kfree(offload_work); } EXPORT_IF_KUNIT(dm_handle_hpd_rx_offload_work); struct hpd_rx_irq_offload_work_queue *amdgpu_dm_hpd_rx_irq_create_workqueue(struct amdgpu_device *adev) { struct dc *dc = adev->dm.dc; int max_caps = dc->caps.max_links; int i = 0; struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL; hpd_rx_offload_wq = kzalloc_objs(*hpd_rx_offload_wq, max_caps); if (!hpd_rx_offload_wq) return NULL; for (i = 0; i < max_caps; i++) { hpd_rx_offload_wq[i].wq = create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq"); if (hpd_rx_offload_wq[i].wq == NULL) { drm_err(adev_to_drm(adev), "create amdgpu_dm_hpd_rx_offload_wq fail!"); goto out_err; } spin_lock_init(&hpd_rx_offload_wq[i].offload_lock); } return hpd_rx_offload_wq; out_err: for (i = 0; i < max_caps; i++) { if (hpd_rx_offload_wq[i].wq) destroy_workqueue(hpd_rx_offload_wq[i].wq); } kfree(hpd_rx_offload_wq); return NULL; } EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_create_workqueue); void amdgpu_dm_hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm) { int i; if (dm->hpd_rx_offload_wq) { for (i = 0; i < dm->dc->caps.max_links; i++) flush_workqueue(dm->hpd_rx_offload_wq[i].wq); } } EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_work_suspend); STATIC_IFN_KUNIT bool are_sinks_equal(const struct dc_sink *sink1, const struct dc_sink *sink2) { if (!sink1 || !sink2) return false; if (sink1->sink_signal != sink2->sink_signal) return false; if (sink1->dc_edid.length != sink2->dc_edid.length) return false; if (memcmp(sink1->dc_edid.raw_edid, sink2->dc_edid.raw_edid, sink1->dc_edid.length) != 0) return false; return true; } EXPORT_IF_KUNIT(are_sinks_equal); /** * DOC: amdgpu_dm_hdmi_hpd_debounce_work * * HDMI HPD debounce delay in milliseconds. When an HDMI display toggles HPD * (such as during power save transitions), this delay determines how long to * wait before processing the HPD event. This allows distinguishing between a * physical unplug (>hdmi_hpd_debounce_delay) * and a spontaneous RX HPD toggle (base; struct drm_device *dev = connector->dev; struct amdgpu_device *adev = drm_to_adev(dev); struct dc *dc = aconnector->dc_link->ctx->dc; bool fake_reconnect = false; bool reallow_idle = false; bool ret = false; guard(mutex)(&aconnector->hpd_lock); /* Re-detect the display */ scoped_guard(mutex, &adev->dm.dc_lock) { if (dc->caps.ips_support && dc->ctx->dmub_srv->idle_allowed) { dc_allow_idle_optimizations(dc, false); reallow_idle = true; } ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD); } if (ret) { /* Apply workaround delay for certain panels */ amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink); /* Compare sinks to determine if this was a spontaneous HPD toggle */ if (are_sinks_equal(aconnector->dc_link->local_sink, aconnector->hdmi_prev_sink)) { /* * Sinks match - this was a spontaneous HDMI HPD toggle. */ drm_dbg_kms(dev, "HDMI HPD: Sink unchanged after debounce, internal re-enable\n"); fake_reconnect = true; } /* Update connector state */ amdgpu_dm_update_connector_after_detect(aconnector); drm_modeset_lock_all(dev); dm_restore_drm_connector_state(dev, connector); drm_modeset_unlock_all(dev); /* Only notify OS if sink actually changed */ if (!fake_reconnect && aconnector->base.force == DRM_FORCE_UNSPECIFIED) drm_kms_helper_hotplug_event(dev); } /* Release the cached sink reference */ if (aconnector->hdmi_prev_sink) { dc_sink_release(aconnector->hdmi_prev_sink); aconnector->hdmi_prev_sink = NULL; } scoped_guard(mutex, &adev->dm.dc_lock) { if (reallow_idle && dc->caps.ips_support) dc_allow_idle_optimizations(dc, true); } } EXPORT_IF_KUNIT(amdgpu_dm_hdmi_hpd_debounce_work); STATIC_IFN_KUNIT void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector, enum dc_detect_reason reason) { struct drm_connector *connector = &aconnector->base; struct drm_device *dev = connector->dev; enum dc_connection_type new_connection_type = dc_connection_none; struct amdgpu_device *adev = drm_to_adev(dev); struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state); struct dc *dc = aconnector->dc_link->ctx->dc; bool ret = false; bool debounce_required = false; if (adev->dm.disable_hpd_irq) return; /* * In case of failure or MST no need to update connector status or notify the OS * since (for MST case) MST does this in its own context. */ guard(mutex)(&aconnector->hpd_lock); if (adev->dm.hdcp_workqueue) { hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index); dm_con_state->update_hdcp = true; } if (aconnector->fake_enable) aconnector->fake_enable = false; aconnector->timing_changed = false; if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type)) drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n"); /* * Check for HDMI disconnect with debounce enabled. */ debounce_required = (aconnector->hdmi_hpd_debounce_delay_ms > 0 && dc_is_hdmi_signal(aconnector->dc_link->connector_signal) && new_connection_type == dc_connection_none && aconnector->dc_link->local_sink != NULL); if (aconnector->base.force && new_connection_type == dc_connection_none) { amdgpu_dm_emulated_link_detect(aconnector->dc_link); drm_modeset_lock_all(dev); dm_restore_drm_connector_state(dev, connector); drm_modeset_unlock_all(dev); if (aconnector->base.force == DRM_FORCE_UNSPECIFIED || reason == DETECT_REASON_HPDRX) drm_kms_helper_connector_hotplug_event(connector); } else if (debounce_required) { /* * HDMI disconnect detected - schedule delayed work instead of * processing immediately. This allows us to coalesce spurious * HDMI signals from physical unplugs. */ drm_dbg_kms(dev, "HDMI HPD: Disconnect detected, scheduling debounce work (%u ms)\n", aconnector->hdmi_hpd_debounce_delay_ms); /* Cache the current sink for later comparison */ if (aconnector->hdmi_prev_sink) dc_sink_release(aconnector->hdmi_prev_sink); aconnector->hdmi_prev_sink = aconnector->dc_link->local_sink; if (aconnector->hdmi_prev_sink) dc_sink_retain(aconnector->hdmi_prev_sink); /* Schedule delayed detection. */ if (mod_delayed_work(system_percpu_wq, &aconnector->hdmi_hpd_debounce_work, msecs_to_jiffies(aconnector->hdmi_hpd_debounce_delay_ms))) drm_dbg_kms(dev, "HDMI HPD: Re-scheduled debounce work\n"); } else { /* If the aconnector->hdmi_hpd_debounce_work is scheduled, exit early */ if (delayed_work_pending(&aconnector->hdmi_hpd_debounce_work)) return; scoped_guard(mutex, &adev->dm.dc_lock) { dc_exit_ips_for_hw_access(dc); ret = dc_link_detect(aconnector->dc_link, reason); } if (ret) { /* w/a delay for certain panels */ amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink); amdgpu_dm_update_connector_after_detect(aconnector); drm_modeset_lock_all(dev); dm_restore_drm_connector_state(dev, connector); drm_modeset_unlock_all(dev); if (aconnector->base.force == DRM_FORCE_UNSPECIFIED || reason == DETECT_REASON_HPDRX) drm_kms_helper_connector_hotplug_event(connector); } } } EXPORT_IF_KUNIT(handle_hpd_irq_helper); STATIC_IFN_KUNIT void handle_hpd_irq(void *param) { struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param; handle_hpd_irq_helper(aconnector, DETECT_REASON_HPD); } EXPORT_IF_KUNIT(handle_hpd_irq); STATIC_IFN_KUNIT void schedule_hpd_rx_offload_work(struct amdgpu_device *adev, struct hpd_rx_irq_offload_work_queue *offload_wq, union hpd_irq_data hpd_irq_data) { struct hpd_rx_irq_offload_work *offload_work = kzalloc_obj(*offload_work); if (!offload_work) { drm_err(adev_to_drm(adev), "Failed to allocate hpd_rx_irq_offload_work.\n"); return; } INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work); offload_work->data = hpd_irq_data; offload_work->offload_wq = offload_wq; offload_work->adev = adev; queue_work(offload_wq->wq, &offload_work->work); drm_dbg_kms(adev_to_drm(adev), "queue work to handle hpd_rx offload work"); } EXPORT_IF_KUNIT(schedule_hpd_rx_offload_work); STATIC_IFN_KUNIT void handle_hpd_rx_irq(void *param) { struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param; struct drm_connector *connector = &aconnector->base; struct drm_device *dev = connector->dev; struct dc_link *dc_link = aconnector->dc_link; bool is_mst_root_connector = aconnector->mst_mgr.mst_state; bool result = false; enum dc_connection_type new_connection_type = dc_connection_none; struct amdgpu_device *adev = drm_to_adev(dev); union hpd_irq_data hpd_irq_data; bool link_loss = false; bool has_left_work = false; int idx = dc_link->link_index; struct hpd_rx_irq_offload_work_queue *offload_wq = &adev->dm.hpd_rx_offload_wq[idx]; struct dc *dc = aconnector->dc_link->ctx->dc; memset(&hpd_irq_data, 0, sizeof(hpd_irq_data)); if (adev->dm.disable_hpd_irq) return; /* * TODO:Temporary add mutex to protect hpd interrupt not have a gpio * conflict, after implement i2c helper, this mutex should be * retired. */ mutex_lock(&aconnector->hpd_lock); result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data, &link_loss, true, &has_left_work); if (!has_left_work) goto out; if (hpd_irq_data.bytes.device_service_irq.bits.AUTOMATED_TEST) { schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data); goto out; } if (dc_link_dp_allow_hpd_rx_irq(dc_link)) { if (hpd_irq_data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || hpd_irq_data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { bool skip = false; /* * DOWN_REP_MSG_RDY is also handled by polling method * mgr->cbs->poll_hpd_irq() */ spin_lock(&offload_wq->offload_lock); skip = offload_wq->is_handling_mst_msg_rdy_event; if (!skip) offload_wq->is_handling_mst_msg_rdy_event = true; spin_unlock(&offload_wq->offload_lock); if (!skip) schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data); goto out; } if (link_loss) { bool skip = false; spin_lock(&offload_wq->offload_lock); skip = offload_wq->is_handling_link_loss; if (!skip) offload_wq->is_handling_link_loss = true; spin_unlock(&offload_wq->offload_lock); if (!skip) schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data); goto out; } } out: if (result && !is_mst_root_connector) { /* Downstream Port status changed. */ if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n"); if (aconnector->base.force && new_connection_type == dc_connection_none) { amdgpu_dm_emulated_link_detect(dc_link); if (aconnector->fake_enable) aconnector->fake_enable = false; amdgpu_dm_update_connector_after_detect(aconnector); drm_modeset_lock_all(dev); dm_restore_drm_connector_state(dev, connector); drm_modeset_unlock_all(dev); drm_kms_helper_connector_hotplug_event(connector); } else { bool ret = false; mutex_lock(&adev->dm.dc_lock); dc_exit_ips_for_hw_access(dc); ret = dc_link_detect(dc_link, DETECT_REASON_HPDRX); mutex_unlock(&adev->dm.dc_lock); if (ret) { if (aconnector->fake_enable) aconnector->fake_enable = false; amdgpu_dm_update_connector_after_detect(aconnector); drm_modeset_lock_all(dev); dm_restore_drm_connector_state(dev, connector); drm_modeset_unlock_all(dev); drm_kms_helper_connector_hotplug_event(connector); } } } if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) { if (adev->dm.hdcp_workqueue) hdcp_handle_cpirq(adev->dm.hdcp_workqueue, aconnector->base.index); } if (dc_link->type != dc_connection_mst_branch) drm_dp_cec_irq(&aconnector->dm_dp_aux.aux); mutex_unlock(&aconnector->hpd_lock); } EXPORT_IF_KUNIT(handle_hpd_rx_irq); /** * dmub_hpd_callback - DMUB HPD interrupt processing callback. * @adev: amdgpu_device pointer * @notify: dmub notification structure * * Dmub Hpd interrupt processing callback. Gets displayindex through the * ink index and calls helper to do the processing. */ STATIC_IFN_KUNIT void dmub_hpd_callback(struct amdgpu_device *adev, struct dmub_notification *notify) { struct amdgpu_dm_connector *aconnector; struct amdgpu_dm_connector *hpd_aconnector = NULL; struct drm_connector *connector; struct drm_connector_list_iter iter; struct dc_link *link; u8 link_index = 0; struct drm_device *dev; if (adev == NULL) return; if (notify == NULL) { drm_err(adev_to_drm(adev), "DMUB HPD callback notification was NULL"); return; } if (notify->link_index > adev->dm.dc->link_count) { drm_err(adev_to_drm(adev), "DMUB HPD index (%u)is abnormal", notify->link_index); return; } /* Skip DMUB HPD IRQ in suspend/resume. We will probe them later. */ if (notify->type == DMUB_NOTIFICATION_HPD && adev->in_suspend) { drm_info(adev_to_drm(adev), "Skip DMUB HPD IRQ callback in suspend/resume\n"); return; } link_index = notify->link_index; link = adev->dm.dc->links[link_index]; dev = adev->dm.ddev; drm_connector_list_iter_begin(dev, &iter); drm_for_each_connector_iter(connector, &iter) { if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) continue; aconnector = to_amdgpu_dm_connector(connector); if (link && aconnector->dc_link == link) { if (notify->type == DMUB_NOTIFICATION_HPD) drm_info(adev_to_drm(adev), "DMUB HPD IRQ callback: link_index=%u\n", link_index); else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) drm_info(adev_to_drm(adev), "DMUB HPD RX IRQ callback: link_index=%u\n", link_index); else drm_warn(adev_to_drm(adev), "DMUB Unknown HPD callback type %d, link_index=%u\n", notify->type, link_index); hpd_aconnector = aconnector; break; } } drm_connector_list_iter_end(&iter); if (hpd_aconnector) { if (notify->type == DMUB_NOTIFICATION_HPD) { if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG)) drm_warn(adev_to_drm(adev), "DMUB reported hpd status unchanged. link_index=%u\n", link_index); handle_hpd_irq_helper(hpd_aconnector, DETECT_REASON_HPD); } else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) { handle_hpd_rx_irq(hpd_aconnector); } } } EXPORT_IF_KUNIT(dmub_hpd_callback); /** * dmub_hpd_sense_callback - DMUB HPD sense processing callback. * @adev: amdgpu_device pointer * @notify: dmub notification structure * * HPD sense changes can occur during low power states and need to be * notified from firmware to driver. */ STATIC_IFN_KUNIT void dmub_hpd_sense_callback(struct amdgpu_device *adev, struct dmub_notification *notify) { drm_dbg_driver(adev_to_drm(adev), "DMUB HPD SENSE callback.\n"); } EXPORT_IF_KUNIT(dmub_hpd_sense_callback); int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev) { struct drm_device *dev = adev_to_drm(adev); struct drm_connector *connector; struct amdgpu_dm_connector *aconnector; const struct dc_link *dc_link; struct dc_interrupt_params int_params = {0}; int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; if (dc_is_dmub_outbox_supported(adev->dm.dc)) { if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD, dmub_hpd_callback, true)) { drm_err(adev_to_drm(adev), "fail to register dmub hpd callback"); return -EINVAL; } if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_IRQ, dmub_hpd_callback, true)) { drm_err(adev_to_drm(adev), "fail to register dmub hpd callback"); return -EINVAL; } if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_SENSE_NOTIFY, dmub_hpd_sense_callback, true)) { drm_err(adev_to_drm(adev), "fail to register dmub hpd sense callback"); return -EINVAL; } } list_for_each_entry(connector, &dev->mode_config.connector_list, head) { if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) continue; aconnector = to_amdgpu_dm_connector(connector); dc_link = aconnector->dc_link; if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) { int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; int_params.irq_source = dc_link->irq_source_hpd; if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_HPD1 || int_params.irq_source > DC_IRQ_SOURCE_HPD6) { drm_err(adev_to_drm(adev), "Failed to register hpd irq %u for %s!\n", int_params.irq_source, connector->name); return -EINVAL; } if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, handle_hpd_irq, (void *) aconnector)) return -ENOMEM; } if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) { /* Also register for DP short pulse (hpd_rx). */ int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; int_params.irq_source = dc_link->irq_source_hpd_rx; if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_HPD1RX || int_params.irq_source > DC_IRQ_SOURCE_HPD6RX) { drm_err(adev_to_drm(adev), "Failed to register hpd rx irq %u for %s!\n", int_params.irq_source, connector->name); return -EINVAL; } if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, handle_hpd_rx_irq, (void *) aconnector)) return -ENOMEM; } } return 0; } EXPORT_IF_KUNIT(amdgpu_dm_register_hpd_handlers); /* ========== IRQ handlers ========== */ struct amdgpu_crtc * amdgpu_dm_get_crtc_by_otg_inst(struct amdgpu_device *adev, int otg_inst) { struct drm_device *dev = adev_to_drm(adev); struct drm_crtc *crtc; struct amdgpu_crtc *amdgpu_crtc; if (WARN_ON(otg_inst == -1)) return adev->mode_info.crtcs[0]; list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { amdgpu_crtc = to_amdgpu_crtc(crtc); if (amdgpu_crtc->otg_inst == otg_inst) return amdgpu_crtc; } return NULL; } EXPORT_IF_KUNIT(amdgpu_dm_get_crtc_by_otg_inst); /** * dm_pflip_high_irq() - Handle pageflip interrupt * @interrupt_params: ignored * * Handles the pageflip interrupt by notifying all interested parties * that the pageflip has been completed. */ STATIC_IFN_KUNIT void dm_pflip_high_irq(void *interrupt_params) { struct amdgpu_crtc *amdgpu_crtc; struct common_irq_params *irq_params = interrupt_params; struct amdgpu_device *adev = irq_params->adev; struct drm_device *dev = adev_to_drm(adev); unsigned long flags; struct drm_pending_vblank_event *e; u32 vpos, hpos, v_blank_start, v_blank_end; bool vrr_active; amdgpu_crtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP); /* IRQ could occur when in initial stage */ /* TODO work and BO cleanup */ if (amdgpu_crtc == NULL) { drm_dbg_state(dev, "CRTC is null, returning.\n"); return; } spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { drm_dbg_state(dev, "amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n", amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED, amdgpu_crtc->crtc_id, amdgpu_crtc); spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); return; } /* page flip completed. */ e = amdgpu_crtc->event; amdgpu_crtc->event = NULL; WARN_ON(!e); vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc); /* Fixed refresh rate, or VRR scanout position outside front-porch? */ if (!vrr_active || !dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start, &v_blank_end, &hpos, &vpos) || (vpos < v_blank_start)) { /* Update to correct count and vblank timestamp if racing with * vblank irq. This also updates to the correct vblank timestamp * even in VRR mode, as scanout is past the front-porch atm. */ drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); /* Wake up userspace by sending the pageflip event with proper * count and timestamp of vblank of flip completion. */ if (e) { drm_crtc_send_vblank_event(&amdgpu_crtc->base, e); /* Event sent, so done with vblank for this flip */ drm_crtc_vblank_put(&amdgpu_crtc->base); } } else if (e) { /* VRR active and inside front-porch: vblank count and * timestamp for pageflip event will only be up to date after * drm_crtc_handle_vblank() has been executed from late vblank * irq handler after start of back-porch (vline 0). We queue the * pageflip event for send-out by drm_crtc_handle_vblank() with * updated timestamp and count, once it runs after us. * * We need to open-code this instead of using the helper * drm_crtc_arm_vblank_event(), as that helper would * call drm_crtc_accurate_vblank_count(), which we must * not call in VRR mode while we are in front-porch! */ /* sequence will be replaced by real count during send-out. */ e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base); e->pipe = amdgpu_crtc->crtc_id; list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list); e = NULL; } /* Keep track of vblank of this flip for flip throttling. We use the * cooked hw counter, as that one incremented at start of this vblank * of pageflip completion, so last_flip_vblank is the forbidden count * for queueing new pageflips if vsync + VRR is enabled. */ amdgpu_crtc->dm_irq_params.last_flip_vblank = amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base); amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE; spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); drm_dbg_state(dev, "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n", amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e); } EXPORT_IF_KUNIT(dm_pflip_high_irq); STATIC_IFN_KUNIT void dm_handle_vmin_vmax_update(struct work_struct *offload_work) { struct vupdate_offload_work *work = container_of(offload_work, struct vupdate_offload_work, work); struct amdgpu_device *adev = work->adev; struct dc_stream_state *stream = work->stream; struct dc_crtc_timing_adjust *adjust = work->adjust; mutex_lock(&adev->dm.dc_lock); dc_stream_adjust_vmin_vmax(adev->dm.dc, stream, adjust); mutex_unlock(&adev->dm.dc_lock); dc_stream_release(stream); kfree(work->adjust); kfree(work); } EXPORT_IF_KUNIT(dm_handle_vmin_vmax_update); static void schedule_dc_vmin_vmax(struct amdgpu_device *adev, struct dc_stream_state *stream, struct dc_crtc_timing_adjust *adjust) { struct vupdate_offload_work *offload_work = kzalloc_obj(*offload_work, GFP_NOWAIT); if (!offload_work) { drm_dbg_driver(adev_to_drm(adev), "Failed to allocate vupdate_offload_work\n"); return; } struct dc_crtc_timing_adjust *adjust_copy = kzalloc_obj(*adjust_copy, GFP_NOWAIT); if (!adjust_copy) { drm_dbg_driver(adev_to_drm(adev), "Failed to allocate adjust_copy\n"); kfree(offload_work); return; } dc_stream_retain(stream); memcpy(adjust_copy, adjust, sizeof(*adjust_copy)); INIT_WORK(&offload_work->work, dm_handle_vmin_vmax_update); offload_work->adev = adev; offload_work->stream = stream; offload_work->adjust = adjust_copy; queue_work(adev->dm.vmin_vmax_wq, &offload_work->work); } /** * dm_crtc_high_irq_handler() - Common OTG vblank/flip event handling * @adev: amdgpu device * @acrtc: the CRTC to service * Performs writeback completion, vblank event handling, CRC processing, VRR BTR * updates and pageflip completion delivery. * * On DCN this is driven by VUPDATE_NO_LOCK (the register latch point) from * dm_vupdate_high_irq(); on DCE it is driven by VLINE0 at the start of vblank * from dm_crtc_high_irq(). */ static void dm_crtc_high_irq_handler(struct amdgpu_device *adev, struct amdgpu_crtc *acrtc) { unsigned long flags; int vrr_active; bool is_dcn = amdgpu_ip_version(adev, DCE_HWIP, 0) != 0; if (acrtc->wb_conn && acrtc->wb_pending) { if (acrtc->wb_frame_done) { /* * Second vblank: the DMA for the captured frame has * had a full frame period to flush to memory. Signal * the out fence now. */ amdgpu_dm_crtc_complete_writeback(acrtc); } else { /* * First vblank after arming: the frame has been * scanned out and the DMA is finishing. Disable * writeback immediately to prevent the hardware from * starting a new capture that would overwrite the * buffer. Signal completion on the next vblank to * ensure the DMA is fully flushed to memory. */ dc_stream_fc_disable_writeback(adev->dm.dc, acrtc->dm_irq_params.stream, 0); acrtc->wb_frame_done = true; } } vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); drm_dbg_vbl(adev_to_drm(adev), "crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id, vrr_active, acrtc->dm_irq_params.active_planes); /** * Core vblank handling. * * On DCN this handler runs at VUPDATE_NO_LOCK, the register latch * point, which is the correct place to timestamp both VRR and non-VRR * vblanks. * * On DCE this handler runs at the start of front-porch, where only * non-VRR timestamping is valid; VRR vblank is deferred to * dm_vupdate_high_irq() after end of front-porch. */ if (is_dcn || !vrr_active) amdgpu_dm_crtc_handle_vblank(acrtc); /** * Following stuff must happen at start of vblank, for crc * computation and below-the-range btr support in vrr mode. */ amdgpu_dm_crtc_handle_crc_irq(&acrtc->base); /* BTR updates need to happen before VUPDATE on Vega and above. */ if (adev->family < AMDGPU_FAMILY_AI) return; spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); if (acrtc->dm_irq_params.stream && acrtc->dm_irq_params.vrr_params.supported) { bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE; mod_freesync_handle_v_update(adev->dm.freesync_module, acrtc->dm_irq_params.stream, &acrtc->dm_irq_params.vrr_params); /* update vmin_vmax only if freesync is enabled, or only if PSR and REPLAY are disabled */ if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { schedule_dc_vmin_vmax(adev, acrtc->dm_irq_params.stream, &acrtc->dm_irq_params.vrr_params.adjust); } } /* * Deliver pageflip completion events (DCN only). * * Since GRPH_PFLIP is not used, VUPDATE_NO_LOCK is the flip latch * point. Deliver any pending pageflip completion event from here, * once HW has consumed the new address (the OTG no longer reports a * pending flip). * * Also handle the case here where there aren't any active planes and * DCN HUBP may be clock-gated, so the flip-pending status may be * undefined. */ if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && acrtc->event) { if (!dc_get_flip_pending_on_otg(adev->dm.dc, acrtc->otg_inst)) { drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); acrtc->event = NULL; drm_crtc_vblank_put(&acrtc->base); acrtc->pflip_status = AMDGPU_FLIP_NONE; } /* * If the flip is still pending, leave it armed and * retry on the next vupdate. */ } else if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && acrtc->dm_irq_params.active_planes == 0) { if (acrtc->event) { drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); acrtc->event = NULL; drm_crtc_vblank_put(&acrtc->base); } acrtc->pflip_status = AMDGPU_FLIP_NONE; } spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } STATIC_IFN_KUNIT void dm_vupdate_high_irq(void *interrupt_params) { struct common_irq_params *irq_params = interrupt_params; struct amdgpu_device *adev = irq_params->adev; struct amdgpu_crtc *acrtc; struct drm_device *drm_dev; struct drm_vblank_crtc *vblank; ktime_t frame_duration_ns, previous_timestamp; unsigned long flags; int vrr_active; acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); if (!acrtc) return; vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); drm_dev = acrtc->base.dev; vblank = drm_crtc_vblank_crtc(&acrtc->base); previous_timestamp = atomic64_read(&irq_params->previous_timestamp); frame_duration_ns = vblank->time - previous_timestamp; if (frame_duration_ns > 0) { trace_amdgpu_refresh_rate_track(acrtc->base.index, frame_duration_ns, ktime_divns(NSEC_PER_SEC, frame_duration_ns)); atomic64_set(&irq_params->previous_timestamp, vblank->time); } drm_dbg_vbl(drm_dev, "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, vrr_active); /* * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver * vblank and pageflip completion events; VSTARTUP and GRPH_PFLIP are * not used. Run the full handler here. */ if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) { dm_crtc_high_irq_handler(adev, acrtc); return; } /* DCE only below. */ /* Core vblank handling is done here after end of front-porch in * vrr mode, as vblank timestamping will give valid results * while now done after front-porch. This will also deliver * page-flip completion events that have been queued to us * if a pageflip happened inside front-porch. */ if (vrr_active && acrtc->dm_irq_params.stream) { bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE; amdgpu_dm_crtc_handle_vblank(acrtc); /* BTR processing for pre-DCE12 ASICs */ if (adev->family < AMDGPU_FAMILY_AI) { spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); mod_freesync_handle_v_update( adev->dm.freesync_module, acrtc->dm_irq_params.stream, &acrtc->dm_irq_params.vrr_params); if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { schedule_dc_vmin_vmax(adev, acrtc->dm_irq_params.stream, &acrtc->dm_irq_params.vrr_params.adjust); } spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } } } EXPORT_IF_KUNIT(dm_vupdate_high_irq); /** * dm_crtc_high_irq() - Handles CRTC interrupt * @interrupt_params: used for determining the CRTC instance * * Handles the CRTC/VSYNC interrupt by notifying DRM's VBLANK event handler. * * Used on DCE (VLINE0, set to vblank start). On DCN the equivalent handling is * driven by VUPDATE_NO_LOCK in dm_vupdate_high_irq(). */ STATIC_IFN_KUNIT void dm_crtc_high_irq(void *interrupt_params) { struct common_irq_params *irq_params = interrupt_params; struct amdgpu_device *adev = irq_params->adev; struct amdgpu_crtc *acrtc; acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); if (!acrtc) return; dm_crtc_high_irq_handler(adev, acrtc); } EXPORT_IF_KUNIT(dm_crtc_high_irq); #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) /** * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for * DCN generation ASICs * @interrupt_params: interrupt parameters * * Used to set crc window/read out crc value at vertical line 0 position */ static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params) { struct common_irq_params *irq_params = interrupt_params; struct amdgpu_device *adev = irq_params->adev; struct amdgpu_crtc *acrtc; acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0); if (!acrtc) return; amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base); } #endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */ STATIC_IFN_KUNIT void dm_handle_hpd_work(struct work_struct *work) { struct dmub_hpd_work *dmub_hpd_wrk; dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work); if (!dmub_hpd_wrk->dmub_notify) { drm_err(adev_to_drm(dmub_hpd_wrk->adev), "dmub_hpd_wrk dmub_notify is NULL"); return; } if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) { dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev, dmub_hpd_wrk->dmub_notify); } kfree(dmub_hpd_wrk->dmub_notify); kfree(dmub_hpd_wrk); } EXPORT_IF_KUNIT(dm_handle_hpd_work); STATIC_IFN_KUNIT const char *dmub_notification_type_str(enum dmub_notification_type e) { switch (e) { case DMUB_NOTIFICATION_NO_DATA: return "NO_DATA"; case DMUB_NOTIFICATION_AUX_REPLY: return "AUX_REPLY"; case DMUB_NOTIFICATION_HPD: return "HPD"; case DMUB_NOTIFICATION_HPD_IRQ: return "HPD_IRQ"; case DMUB_NOTIFICATION_SET_CONFIG_REPLY: return "SET_CONFIG_REPLY"; case DMUB_NOTIFICATION_DPIA_NOTIFICATION: return "DPIA_NOTIFICATION"; case DMUB_NOTIFICATION_HPD_SENSE_NOTIFY: return "HPD_SENSE_NOTIFY"; case DMUB_NOTIFICATION_FUSED_IO: return "FUSED_IO"; default: return ""; } } EXPORT_IF_KUNIT(dmub_notification_type_str); #define DMUB_TRACE_MAX_READ 64 /** * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt * @interrupt_params: used for determining the Outbox instance * * Handles the Outbox Interrupt * event handler. */ STATIC_IFN_KUNIT void dm_dmub_outbox1_low_irq(void *interrupt_params) { struct dmub_notification notify = {0}; struct common_irq_params *irq_params = interrupt_params; struct amdgpu_device *adev = irq_params->adev; struct amdgpu_display_manager *dm = &adev->dm; struct dmcub_trace_buf_entry entry = { 0 }; u32 count = 0; struct dmub_hpd_work *dmub_hpd_wrk; do { if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) { trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count, entry.param0, entry.param1); drm_dbg_driver(adev_to_drm(adev), "trace_code:%u, tick_count:%u, param0:%u, param1:%u\n", entry.trace_code, entry.tick_count, entry.param0, entry.param1); } else break; count++; } while (count <= DMUB_TRACE_MAX_READ); if (count > DMUB_TRACE_MAX_READ) drm_dbg_driver(adev_to_drm(adev), "Warning : count > DMUB_TRACE_MAX_READ"); if (dc_enable_dmub_notifications(adev->dm.dc) && irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) { do { dc_stat_get_dmub_notification(adev->dm.dc, ¬ify); if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) { drm_err(adev_to_drm(adev), "DM: notify type %d invalid!", notify.type); continue; } if (!dm->dmub_callback[notify.type]) { drm_warn(adev_to_drm(adev), "DMUB notification skipped due to no handler: type=%s\n", dmub_notification_type_str(notify.type)); continue; } if (dm->dmub_thread_offload[notify.type]) { dmub_hpd_wrk = kzalloc_obj(*dmub_hpd_wrk, GFP_ATOMIC); if (!dmub_hpd_wrk) { drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk"); return; } dmub_hpd_wrk->dmub_notify = kmemdup(¬ify, sizeof(struct dmub_notification), GFP_ATOMIC); if (!dmub_hpd_wrk->dmub_notify) { kfree(dmub_hpd_wrk); drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk->dmub_notify"); return; } INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work); dmub_hpd_wrk->adev = adev; queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work); } else { dm->dmub_callback[notify.type](adev, ¬ify); } } while (notify.pending_notification); } } EXPORT_IF_KUNIT(dm_dmub_outbox1_low_irq); /* Register IRQ sources and initialize IRQ callbacks */ int amdgpu_dm_dce110_register_irq_handlers(struct amdgpu_device *adev) { struct dc *dc = adev->dm.dc; struct common_irq_params *c_irq_params; struct dc_interrupt_params int_params = {0}; int r; int i; unsigned int src_id; unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY; /* Use different interrupts for VBLANK on DCE 6 vs. newer. */ const unsigned int vblank_d1 = adev->dm.dc->ctx->dce_version >= DCE_VERSION_8_0 ? VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0 : 1; if (adev->family >= AMDGPU_FAMILY_AI) client_id = SOC15_IH_CLIENTID_DCE; int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; /* * Actions of amdgpu_irq_add_id(): * 1. Register a set() function with base driver. * Base driver will call set() function to enable/disable an * interrupt in DC hardware. * 2. Register amdgpu_dm_irq_handler(). * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts * coming from DC hardware. * amdgpu_dm_irq_handler() will re-direct the interrupt to DC * for acknowledging and handling. */ /* Use VBLANK interrupt */ for (i = 0; i < adev->mode_info.num_crtc; i++) { src_id = vblank_d1 + i; r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->crtc_irq); if (r) { drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n"); return r; } int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; int_params.irq_source = dc_interrupt_to_irq_source(dc, src_id, 0); if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n"); return -EINVAL; } c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; c_irq_params->adev = adev; c_irq_params->irq_src = int_params.irq_source; if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, dm_crtc_high_irq, c_irq_params)) return -ENOMEM; } if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) { /* Use VUPDATE interrupt */ for (i = 0; i < adev->mode_info.num_crtc; i++) { src_id = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT + i * 2; r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->vupdate_irq); if (r) { drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n"); return r; } int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; int_params.irq_source = dc_interrupt_to_irq_source(dc, src_id, 0); if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 || int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) { drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n"); return -EINVAL; } c_irq_params = &adev->dm.vupdate_params[ int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1]; c_irq_params->adev = adev; c_irq_params->irq_src = int_params.irq_source; if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, dm_vupdate_high_irq, c_irq_params)) return -ENOMEM; } } /* Use GRPH_PFLIP interrupt */ for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP; i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) { r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq); if (r) { drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n"); return r; } int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; int_params.irq_source = dc_interrupt_to_irq_source(dc, i, 0); if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n"); return -EINVAL; } c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; c_irq_params->adev = adev; c_irq_params->irq_src = int_params.irq_source; if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, dm_pflip_high_irq, c_irq_params)) return -ENOMEM; } /* HPD */ r = amdgpu_irq_add_id(adev, client_id, VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq); if (r) { drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n"); return r; } r = amdgpu_dm_register_hpd_handlers(adev); return r; } EXPORT_IF_KUNIT(amdgpu_dm_dce110_register_irq_handlers); /* Register IRQ sources and initialize IRQ callbacks */ int amdgpu_dm_dcn10_register_irq_handlers(struct amdgpu_device *adev) { struct dc *dc = adev->dm.dc; struct common_irq_params *c_irq_params; struct dc_interrupt_params int_params = {0}; int r; int i; #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) static const unsigned int vrtl_int_srcid[] = { DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL, DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL, DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL, DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL, DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL, DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL }; #endif int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; /* * Actions of amdgpu_irq_add_id(): * 1. Register a set() function with base driver. * Base driver will call set() function to enable/disable an * interrupt in DC hardware. * 2. Register amdgpu_dm_irq_handler(). * Base driver will call amdgpu_dm_irq_handler() for ALL interrupts * coming from DC hardware. * amdgpu_dm_irq_handler() will re-direct the interrupt to DC * for acknowledging and handling. */ /* Use otg vertical line interrupt */ #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) { r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, vrtl_int_srcid[i], &adev->vline0_irq); if (r) { drm_err(adev_to_drm(adev), "Failed to add vline0 irq id!\n"); return r; } int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; int_params.irq_source = dc_interrupt_to_irq_source(dc, vrtl_int_srcid[i], 0); if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_DC1_VLINE0 || int_params.irq_source > DC_IRQ_SOURCE_DC6_VLINE0) { drm_err(adev_to_drm(adev), "Failed to register vline0 irq!\n"); return -EINVAL; } c_irq_params = &adev->dm.vline0_params[int_params.irq_source - DC_IRQ_SOURCE_DC1_VLINE0]; c_irq_params->adev = adev; c_irq_params->irq_src = int_params.irq_source; if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, dm_dcn_vertical_interrupt0_high_irq, c_irq_params)) return -ENOMEM; } #endif /* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to * the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx * to trigger at end of each vblank, regardless of state of the lock, * matching DCE behaviour. */ for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT; i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1; i++) { r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq); if (r) { drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n"); return r; } int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; int_params.irq_source = dc_interrupt_to_irq_source(dc, i, 0); if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 || int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) { drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n"); return -EINVAL; } c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1]; c_irq_params->adev = adev; c_irq_params->irq_src = int_params.irq_source; if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, dm_vupdate_high_irq, c_irq_params)) return -ENOMEM; } /* HPD */ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT, &adev->hpd_irq); if (r) { drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n"); return r; } r = amdgpu_dm_register_hpd_handlers(adev); return r; } EXPORT_IF_KUNIT(amdgpu_dm_dcn10_register_irq_handlers); /* Register Outbox IRQ sources and initialize IRQ callbacks */ int amdgpu_dm_register_outbox_irq_handlers(struct amdgpu_device *adev) { struct dc *dc = adev->dm.dc; struct common_irq_params *c_irq_params; struct dc_interrupt_params int_params = {0}; int r, i; int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT; int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT; r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT, &adev->dmub_outbox_irq); if (r) { drm_err(adev_to_drm(adev), "Failed to add outbox irq id!\n"); return r; } if (dc->ctx->dmub_srv) { i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT; int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT; int_params.irq_source = dc_interrupt_to_irq_source(dc, i, 0); c_irq_params = &adev->dm.dmub_outbox_params[0]; c_irq_params->adev = adev; c_irq_params->irq_src = int_params.irq_source; if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, dm_dmub_outbox1_low_irq, c_irq_params)) return -ENOMEM; } return 0; } EXPORT_IF_KUNIT(amdgpu_dm_register_outbox_irq_handlers);